diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..34c400805e2eb370cbf0d99e32fb7212ddd49877 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name.far filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..7d4bb82e2244f4a942d62af6ad2bcca60996f5dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +.gradio/ +.venv/ +venv/ +*.wav +.DS_Store diff --git a/README.md b/README.md index cc688863b5cf4e77bb0995ebe8cfaf03c3dae1d6..b08cdeefa72f5d859390b215893ef17366d3682e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,18 @@ --- -title: Zonos2 -emoji: 📚 -colorFrom: purple -colorTo: gray +title: ZONOS2 +emoji: 🔊 +colorFrom: blue +colorTo: indigo sdk: gradio -sdk_version: 6.18.0 -python_version: '3.13' +sdk_version: 6.10.0 app_file: app.py -pinned: false +short_description: ZeroGPU text-to-speech with ZONOS2 +python_version: "3.10" +startup_duration_timeout: 1h +preload_from_hub: + - Zyphra/ZONOS2 --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# ZONOS2 on ZeroGPU + +Gradio Space for [Zyphra/ZONOS2](https://huggingface.co/Zyphra/ZONOS2), using the upstream [ZONOS2](https://github.com/Zyphra/ZONOS2) inference code with small ZeroGPU compatibility patches. diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6d6b04c8ebeaa66fd154c50d177f610160265dcb --- /dev/null +++ b/app.py @@ -0,0 +1,317 @@ +import os +import tempfile +import threading +import time +import wave +from pathlib import Path + +os.environ.setdefault("HF_HOME", "/data/.cache/huggingface") +os.environ.setdefault("HF_MODULES_CACHE", "/tmp/hf_modules") +os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib") +os.environ.setdefault("ZONOS2_TTS_NORM_CACHE_DIR", "/tmp/zonos2-tts-norm") +os.environ.setdefault("GRADIO_SSR_MODE", "false") +os.environ.setdefault("NUMBA_DISABLE_CUDA", "1") +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + +for cache_dir in ( + os.environ["HF_HOME"], + os.environ["HF_MODULES_CACHE"], + os.environ["MPLCONFIGDIR"], + os.environ["ZONOS2_TTS_NORM_CACHE_DIR"], +): + Path(cache_dir).mkdir(parents=True, exist_ok=True) + +import spaces +import gradio as gr +import numpy as np +import torch + +from zonos2.message import TTSSamplingParams +from zonos2.tokenizer.textnorm import SERVER_TO_NEMO_LANG, TTSTextNormalizer +from zonos2.tts import TTSLLM + +MODEL_ID = "Zyphra/ZONOS2" +SAMPLE_RATE = 44100 +LANGUAGES = [ + ("English (US)", "en_us"), + ("English (UK)", "en_gb"), + ("French", "fr_fr"), + ("German", "de"), + ("Spanish", "es"), + ("Italian", "it"), + ("Portuguese (Brazil)", "pt_br"), + ("Japanese", "ja"), + ("Mandarin Chinese", "cmn"), + ("Korean", "ko"), +] +SPEAKING_RATE_BUCKETS = [ + ("Default", "default"), + ("Very slow", "0"), + ("Slow", "1"), + ("Relaxed", "2"), + ("Natural", "3"), + ("Bright", "4"), + ("Fast", "5"), + ("Very fast", "6"), + ("Extreme", "7"), +] + +torch.backends.cuda.matmul.allow_tf32 = True + + +def _load_model() -> TTSLLM: + print(f"Loading {MODEL_ID} for ZeroGPU inference...", flush=True) + started = time.perf_counter() + model = TTSLLM( + model_path=MODEL_ID, + decode_audio=True, + cuda_graph_max_bs=0, + max_running_req=4, + max_extend_tokens=4096, + memory_ratio=0.75, + use_pynccl=False, + ) + elapsed = time.perf_counter() - started + print(f"Loaded {MODEL_ID} in {elapsed:.1f}s", flush=True) + return model + + +TTS = _load_model() +TEXT_NORMALIZER = TTSTextNormalizer() +TTS_LOCK = threading.Lock() + + +def _estimate_duration(*args, **kwargs) -> int: + max_tokens = kwargs.get("max_tokens") + if max_tokens is None and len(args) > 4: + max_tokens = args[4] + try: + max_tokens = int(max_tokens) + except (TypeError, ValueError): + max_tokens = 768 + return min(180, max(60, 45 + max_tokens // 12)) + + +def _pcm_float32_to_wav(audio_bytes: bytes, sample_rate: int = SAMPLE_RATE) -> str: + audio = np.frombuffer(audio_bytes, dtype=np.float32) + if audio.size == 0: + raise gr.Error("The model returned no audio. Try increasing max tokens.") + audio = np.nan_to_num(audio, nan=0.0, posinf=0.0, neginf=0.0) + audio = np.clip(audio, -1.0, 1.0) + audio_i16 = (audio * 32767.0).astype(np.int16) + + handle = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + handle.close() + with wave.open(handle.name, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(audio_i16.tobytes()) + return handle.name + + +def _normalize_text(text: str, language: str, enabled: bool) -> str: + if not enabled: + return text + if language not in SERVER_TO_NEMO_LANG: + return text + return TEXT_NORMALIZER.normalize(text, language) + + +def _speaking_rate_bucket(value: str) -> int | None: + if value == "default": + return None + return int(value) + + +@spaces.GPU(duration=_estimate_duration) +def synthesize( + text: str, + language: str, + text_normalization: bool, + speaking_rate: str, + max_tokens: int, + temperature: float, + topk: int, + top_p: float, + min_p: float, + repetition_penalty: float, + seed: int, +): + text = (text or "").strip() + if not text: + raise gr.Error("Enter text to synthesize.") + if len(text) > 1200: + raise gr.Error("Keep the prompt under 1200 characters for this Space.") + + normalized = _normalize_text(text, language, text_normalization) + params = TTSSamplingParams( + temperature=float(temperature), + topk=int(topk), + top_p=float(top_p), + min_p=float(min_p), + max_tokens=int(max_tokens), + repetition_window=50, + repetition_penalty=float(repetition_penalty), + repetition_codebooks=8, + seed=None if seed is None or int(seed) < 0 else int(seed), + ) + + started = time.perf_counter() + with TTS_LOCK: + torch.cuda.set_stream(TTS.stream) + result = TTS.generate_one( + normalized, + params, + decode_audio=True, + speaking_rate_bucket=_speaking_rate_bucket(speaking_rate), + quality_buckets=None, + ) + elapsed = time.perf_counter() - started + + wav_path = _pcm_float32_to_wav(result["audio"], result.get("sample_rate", SAMPLE_RATE)) + frames = len(result.get("audio_tokens") or []) + eos_frame = result.get("eos_frame") + status = f"Generated {frames} frames in {elapsed:.1f}s" + if eos_frame is not None: + status += f" (EOS frame {eos_frame})" + if normalized != text: + status += f"\n\nNormalized text: {normalized}" + return wav_path, status + + +CSS = """ +main, .gradio-container, .gradio-container > .fillable { + max-width: 1180px !important; + margin-inline: auto !important; +} +.compact-status textarea { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} +""" + + +with gr.Blocks(title="ZONOS2", css=CSS) as demo: + gr.Markdown("# ZONOS2") + with gr.Row(): + with gr.Column(scale=5): + text = gr.Textbox( + label="Text", + value="In the quiet hum of the studio, ZONOS2 turns written words into natural speech.", + lines=6, + max_length=1200, + ) + with gr.Row(): + language = gr.Dropdown( + choices=LANGUAGES, + value="en_us", + label="Language", + ) + speaking_rate = gr.Dropdown( + choices=SPEAKING_RATE_BUCKETS, + value="default", + label="Speaking rate", + ) + text_normalization = gr.Checkbox(value=True, label="Text normalization") + generate = gr.Button("Generate", variant="primary") + with gr.Column(scale=4): + audio = gr.Audio(label="Audio", type="filepath", format="wav") + status = gr.Textbox( + label="Status", + lines=5, + interactive=False, + elem_classes=["compact-status"], + ) + + with gr.Accordion("Sampling", open=False): + with gr.Row(): + max_tokens = gr.Slider( + minimum=128, + maximum=2048, + step=64, + value=768, + label="Max audio tokens", + ) + seed = gr.Number(value=-1, precision=0, label="Seed (-1 random)") + with gr.Row(): + temperature = gr.Slider(0.1, 2.0, value=1.15, step=0.05, label="Temperature") + topk = gr.Slider(1, 512, value=106, step=1, label="Top-k") + with gr.Row(): + top_p = gr.Slider(0.0, 1.0, value=0.0, step=0.01, label="Top-p") + min_p = gr.Slider(0.0, 0.5, value=0.18, step=0.01, label="Min-p") + repetition_penalty = gr.Slider( + 1.0, + 2.0, + value=1.2, + step=0.05, + label="Repetition penalty", + ) + + gr.Examples( + examples=[ + [ + "The first explorers landed just after sunrise, carrying maps, coffee, and impossible optimism.", + "en_us", + True, + "default", + 512, + 1.15, + 106, + 0.0, + 0.18, + 1.2, + -1, + ], + [ + "Le modèle parle avec une voix claire, expressive et naturellement rythmée.", + "fr_fr", + True, + "default", + 512, + 1.15, + 106, + 0.0, + 0.18, + 1.2, + -1, + ], + ], + inputs=[ + text, + language, + text_normalization, + speaking_rate, + max_tokens, + temperature, + topk, + top_p, + min_p, + repetition_penalty, + seed, + ], + ) + + generate.click( + fn=synthesize, + inputs=[ + text, + language, + text_normalization, + speaking_rate, + max_tokens, + temperature, + topk, + top_p, + min_p, + repetition_penalty, + seed, + ], + outputs=[audio, status], + api_name="generate", + concurrency_limit=1, + ) + + +if __name__ == "__main__": + demo.queue(max_size=8, default_concurrency_limit=1).launch() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2814edcfa6791dc3290e1942aaed1d2a79986582 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +gradio>=6.10.0 +spaces>=0.41.0 +numpy +tqdm +msgpack +sgl_kernel>=0.3.17.post1 +descript-audio-codec==1.0.0 +transformers>=4.56.0,<=4.57.3 +huggingface_hub +pyzmq +apache-tvm-ffi>=0.1.4 +nvidia-cutlass-dsl==4.3.1 +flashinfer-cubin==0.5.3 +flashinfer-python>=0.5.3 +flashinfer-jit-cache @ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.5.3/flashinfer_jit_cache-0.5.3+cu128-cp39-abi3-manylinux_2_28_x86_64.whl +ninja>=1.13.0 +kernels>=0.12.1 +pynini==2.1.6 +sacremoses>=0.1.1 diff --git a/zonos2/__main__.py b/zonos2/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..09022c6afcb6a44030c9cf95ad8d2ab031409d85 --- /dev/null +++ b/zonos2/__main__.py @@ -0,0 +1,5 @@ +from .server import launch_server + +assert __name__ == "__main__" + +launch_server() diff --git a/zonos2/attention/__init__.py b/zonos2/attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..897ee4ed799c7066cd17b45ccbf5ade71d0451dc --- /dev/null +++ b/zonos2/attention/__init__.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol + +import torch +from zonos2.utils import Registry, init_logger, is_sm90_supported, is_sm100_supported + +from .base import BaseAttnBackend, BaseAttnMetadata, HybridBackend + +if TYPE_CHECKING: + from zonos2.kvcache import BaseKVCache + from zonos2.models import ModelConfig + +logger = init_logger(__name__) + + +class BackendCreator(Protocol): + def __call__( + self, config: ModelConfig, kvcache: BaseKVCache, page_table: torch.Tensor + ) -> BaseAttnBackend: ... + + +SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend") + + +def resolve_auto_backend(config: ModelConfig) -> str: + """Determine the best attention backend based on the GPU architecture and model.""" + if is_sm100_supported(): # blackwell + return "fi" + elif is_sm90_supported(): # hopper + return "fa,fi" + else: # pre-hopper + return "fi" + + +@SUPPORTED_ATTENTION_BACKENDS.register("fi") +def create_fi_backend(config: ModelConfig, kvcache: BaseKVCache, page_table: torch.Tensor): + from .fi import FlashInferBackend + + return FlashInferBackend(config, kvcache, page_table) + + +@SUPPORTED_ATTENTION_BACKENDS.register("fa") +def create_fa_backend(config: ModelConfig, kvcache: BaseKVCache, page_table: torch.Tensor): + from .fa import FlashAttentionBackend + + return FlashAttentionBackend(config, kvcache, page_table) + + +def validate_backend(backend: str): + if backend != "auto": + required_backends = backend.split(",") if "," in backend else [backend] + supported = SUPPORTED_ATTENTION_BACKENDS.supported_names() + for b in required_backends: + if b not in supported: + from argparse import ArgumentTypeError + + raise ArgumentTypeError( + f"Unsupported attention backend: {b}. Supported backends: {supported}" + ) + return backend + + +def create_attention_backend( + backend: str, + config: ModelConfig, + kvcache: BaseKVCache, + page_table: torch.Tensor, +) -> BaseAttnBackend: + if backend == "auto": + backend = resolve_auto_backend(config) + logger.info(f"Auto-selected attention backend: {backend}") + + if "," in backend: + assert backend.count(",") == 1, "Only one comma is allowed in hybrid backend" + p_backend, d_backend = backend.split(",", 1) + if p_backend != d_backend: + logger.info(f"Using hybrid attention backend: prefill={p_backend}, decode={d_backend}") + p_backend = create_attention_backend(p_backend, config, kvcache, page_table) + d_backend = create_attention_backend(d_backend, config, kvcache, page_table) + return HybridBackend(p_backend, d_backend) + backend = p_backend # both are the same, fall through to single backend + logger.warning(f"P/D attention backends are the same: {backend}, using single backend.") + + return SUPPORTED_ATTENTION_BACKENDS[backend](config, kvcache, page_table) + + +__all__ = [ + "BaseAttnMetadata", + "BaseAttnBackend", + "create_attention_backend", + "SUPPORTED_ATTENTION_BACKENDS", + "validate_backend", +] diff --git a/zonos2/attention/base.py b/zonos2/attention/base.py new file mode 100644 index 0000000000000000000000000000000000000000..430377581769f82027de82b157d3ac66373a9956 --- /dev/null +++ b/zonos2/attention/base.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + import torch + from zonos2.core import TTSBatch + + +@dataclass +class BaseAttnMetadata(ABC): + positions: torch.Tensor + + @abstractmethod + def get_last_indices(self, bs: int) -> torch.Tensor: ... + + +class BaseAttnBackend(ABC): + @abstractmethod + def forward( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, layer_id: int, batch: TTSBatch + ) -> torch.Tensor: ... + + @abstractmethod + def prepare_metadata(self, batch: TTSBatch) -> None: ... + + @abstractmethod + def init_capture_graph( + self, max_seq_len: int, bs_list: List[int], frame_width: int = 1 + ) -> None: ... + + @abstractmethod + def prepare_for_capture(self, batch: TTSBatch) -> None: ... + + @abstractmethod + def prepare_for_replay(self, batch: TTSBatch) -> None: ... + + +class HybridBackend(BaseAttnBackend): + def __init__( + self, + prefill_backend: BaseAttnBackend, + decode_backend: BaseAttnBackend, + ) -> None: + self.prefill_backend = prefill_backend + self.decode_backend = decode_backend + + def forward( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, layer_id: int, batch: TTSBatch + ) -> torch.Tensor: + backend = self.prefill_backend if batch.is_prefill else self.decode_backend + return backend.forward(q, k, v, layer_id, batch) + + def prepare_metadata(self, batch: TTSBatch) -> None: + backend = self.prefill_backend if batch.is_prefill else self.decode_backend + return backend.prepare_metadata(batch) + + def init_capture_graph( + self, max_seq_len: int, bs_list: List[int], frame_width: int = 1 + ) -> None: + self.decode_backend.init_capture_graph(max_seq_len, bs_list, frame_width) + + def prepare_for_capture(self, batch: TTSBatch) -> None: + self.decode_backend.prepare_for_capture(batch) + + def prepare_for_replay(self, batch: TTSBatch) -> None: + self.decode_backend.prepare_for_replay(batch) diff --git a/zonos2/attention/fa.py b/zonos2/attention/fa.py new file mode 100644 index 0000000000000000000000000000000000000000..3cd14f0d7a5bb67ad0e6e8b1904663780b8e10b0 --- /dev/null +++ b/zonos2/attention/fa.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Tuple + +import torch + +from .base import BaseAttnBackend, BaseAttnMetadata +from .utils import BaseCaptureData, make_positions + +if TYPE_CHECKING: + from zonos2.core import TTSBatch + from zonos2.kvcache import BaseKVCache + from zonos2.models import ModelConfig + + +@dataclass +class FACaptureData(BaseCaptureData): + pass + + +@dataclass +class FAMetadata(BaseAttnMetadata): + cu_seqlens_k: torch.Tensor + cu_seqlens_q: torch.Tensor + cache_seqlens: torch.Tensor + max_seqlen_k: int + max_seqlen_q: int + + page_table: torch.Tensor + + def get_positions(self) -> torch.Tensor: + return self.positions + + def get_last_indices(self, bs: int) -> torch.Tensor: + return self.cu_seqlens_q[1 : 1 + bs] - 1 + + +class FlashAttentionBackend(BaseAttnBackend): + def __init__(self, config: ModelConfig, kvcache: BaseKVCache, page_table: torch.Tensor): + self.config = config + self.kvcache = kvcache + self.capture: FACaptureData | None = None + self.max_graph_bs = 0 + self.capture_bs: List[int] = [] + self.scale = config.head_dim**-0.5 + self.page_table = page_table + + def forward( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, layer_id: int, batch: TTSBatch + ) -> torch.Tensor: + metadata = batch.attn_metadata + assert isinstance(metadata, FAMetadata) + self.kvcache.store_kv(k, v, batch.out_loc, layer_id) + return _fa_sgl_impl( + q=q, + k_cache=self.kvcache.k_cache(layer_id), + v_cache=self.kvcache.v_cache(layer_id), + page_table=metadata.page_table, + cache_seqlens=metadata.cache_seqlens, + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k_new=metadata.cu_seqlens_k, + max_seqlen_q=metadata.max_seqlen_q, + softmax_scale=self.scale, + ) + + def prepare_metadata(self, batch: TTSBatch) -> None: + reqs = batch.padded_reqs + + padded_size = len(reqs) + seqlens_q = [req.extend_len for req in reqs] + seqlens_k = [req.device_len for req in reqs] + cached_lens = [req.cached_len for req in reqs] + max_seqlen_k = max(seqlens_k) + max_seqlen_q = max(seqlens_q) + cpu_kwargs = {"device": "cpu", "dtype": torch.int32, "pin_memory": True} + + device = self.kvcache.device + cache_seqlens = torch.tensor(seqlens_k, **cpu_kwargs) + cache_seqlens = cache_seqlens.to(device, non_blocking=True) + cu_seqlens_k = torch.tensor([0] + seqlens_k, **cpu_kwargs).cumsum_(dim=0) + cu_seqlens_k = cu_seqlens_k.to(device, non_blocking=True) + + if max_seqlen_q == 1: + cu_seqlens_q = torch.arange(0, padded_size + 1, device=device, dtype=torch.int32) + elif all(l == 0 for l in cached_lens): # prefill with no cache hit + cu_seqlens_q = cu_seqlens_k + else: # normal extend prefill, with partial cache hit + cu_seqlens_q = torch.tensor([0] + seqlens_q, **cpu_kwargs).cumsum_(dim=0) + cu_seqlens_q = cu_seqlens_q.to(self.kvcache.device, non_blocking=True) + + positions = make_positions(device, reqs) + page_table = self.page_table + new_page_table = torch.stack([page_table[req.table_idx, :max_seqlen_k] for req in reqs]) + + # copy from CPU to GPU + batch.attn_metadata = FAMetadata( + cu_seqlens_k=cu_seqlens_k, + cu_seqlens_q=cu_seqlens_q, + positions=positions, + cache_seqlens=cache_seqlens, + max_seqlen_k=max_seqlen_k, + max_seqlen_q=max_seqlen_q, + page_table=new_page_table, + ) + + def init_capture_graph( + self, max_seq_len: int, bs_list: List[int], frame_width: int = 1 + ) -> None: + assert self.capture is None, "Capture already initialized." + max_bs = max(bs_list) + capture = FACaptureData.create(max_bs, max_seq_len, self.kvcache.device, frame_width) + self.max_graph_bs = max_bs + self.capture = capture + self.capture_bs = sorted(bs_list) + + def prepare_for_capture(self, batch: TTSBatch) -> None: + assert (bs := batch.size) in self.capture_bs and self.capture + capture = self.capture + metadata = FAMetadata( + cu_seqlens_k=capture.cu_seqlens_k[: bs + 1], + cu_seqlens_q=capture.cu_seqlens_q[: bs + 1], + positions=capture.positions[:bs], + cache_seqlens=capture.seq_lens[:bs], + max_seqlen_k=capture.page_table.size(1), + max_seqlen_q=1, # decode only + page_table=capture.page_table[:bs, :], + ) + batch.attn_metadata = metadata + batch.input_ids = capture.input_ids[:bs] + batch.out_loc = capture.out_loc[:bs] + + def prepare_for_replay(self, batch: TTSBatch) -> None: + metadata, bs = batch.attn_metadata, batch.padded_size + assert isinstance(metadata, FAMetadata) + assert self.capture is not None and bs in self.capture_bs + capture = self.capture + + # Copy all dynamic tensors to capture buffers (fixed memory addresses for CUDA graph) + capture.input_ids[:bs].copy_(batch.input_ids) + capture.out_loc[:bs].copy_(batch.out_loc) + capture.positions[:bs].copy_(metadata.positions) + capture.cu_seqlens_k[: bs + 1].copy_(metadata.cu_seqlens_k) + capture.seq_lens[:bs].copy_(metadata.cache_seqlens) + capture.page_table[:bs, : metadata.max_seqlen_k].copy_(metadata.page_table) + + # Update metadata and batch to point to capture buffers + # This ensures CUDA graph uses fixed memory addresses for all inputs + metadata.positions = capture.positions[:bs] + metadata.cu_seqlens_k = capture.cu_seqlens_k[: bs + 1] + metadata.cache_seqlens = capture.seq_lens[:bs] + metadata.page_table = capture.page_table[:bs, :] + batch.input_ids = capture.input_ids[:bs] + batch.out_loc = capture.out_loc[:bs] + + +def _fa_sgl_impl( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + page_table: torch.Tensor, + cache_seqlens: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k_new: torch.Tensor, + max_seqlen_q: int, + softmax_scale: float, + sm_margin: int = 0, + window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window + softcap: float = 0.0, # 0.0 means deactivated + num_splits: int = 0, # Can be tuned for speed + pack_gqa: bool | None = None, # Can be tuned for speed + causal: bool = True, +) -> torch.Tensor: + try: + from sgl_kernel.flash_attn import flash_attn_with_kvcache + except ImportError as e: + raise ImportError( + "sgl_kernel.flash_attn is not found. Please install it with `pip install sgl-kernel`.\n" + "If you're sure it's correctly installed, try `apt update && apt install libnuma1`." + ) from e + + return flash_attn_with_kvcache( # type: ignore + q=q, + k_cache=k_cache, + v_cache=v_cache, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k_new, + max_seqlen_q=max_seqlen_q, + softmax_scale=softmax_scale, + sm_margin=sm_margin, + window_size=window_size, + softcap=softcap, + num_splits=num_splits, + pack_gqa=pack_gqa, + causal=causal, + ver=3, # TODO: support FA4 on blackwell + ) diff --git a/zonos2/attention/fi.py b/zonos2/attention/fi.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b526fad7e65b2ccff64c487ba4172b98f012f3 --- /dev/null +++ b/zonos2/attention/fi.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from functools import cached_property +from typing import TYPE_CHECKING, Dict, List, Literal + +import torch +from zonos2.distributed import get_tp_info +from zonos2.env import ENV +from zonos2.utils import divide_even +from zonos2.utils.logger import init_logger + +from .base import BaseAttnBackend, BaseAttnMetadata +from .utils import BaseCaptureData, make_positions + +if TYPE_CHECKING: + from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + BatchPrefillWithPagedKVCacheWrapper, + CUDAGraphBatchDecodeWithPagedKVCacheWrapper, + ) + from zonos2.core import TTSBatch + from zonos2.kvcache import BaseKVCache + from zonos2.models import ModelConfig + + +def _next_power_of_2(n: int) -> int: + if n <= 1: + return 1 + return 1 << math.ceil(math.log2(n)) + + +logger = init_logger(__name__) + +_fi_debug_counter = 0 + + +@dataclass +class FICaptureData(BaseCaptureData): + @property + def one_tensor(self) -> torch.Tensor: + return self.seq_lens + + @property + def indices(self) -> torch.Tensor: + return self.page_table + + +@dataclass +class FIMetadata(BaseAttnMetadata): + # fmt: off + cu_seqlens_q_cpu: torch.Tensor # on cpu + cu_seqlens_k_cpu: torch.Tensor # on cpu + cu_seqlens_q_gpu: torch.Tensor # on gpu + indices: torch.Tensor # on gpu + last_page_len_cpu: torch.Tensor # on cpu + num_qo_heads: int + num_kv_heads: int + head_dim: int + page_size: Literal[1] # currently only support page_size=1 + pos_encoding_mode: str + seq_lens_cpu: torch.Tensor # on cpu + dtype: torch.dtype + wrapper: BatchPrefillWithPagedKVCacheWrapper | BatchDecodeWithPagedKVCacheWrapper + initialized: bool = False + sm_scale: float = 0.0 # softmax scale (1/sqrt(head_dim)), set during prepare_metadata + # fmt: on + + def __post_init__(self) -> None: + assert self.page_size == 1, "Currently only page_size=1 is supported." + assert ( + self.positions.is_cuda + and self.cu_seqlens_k_cpu.is_cpu + and self.cu_seqlens_q_cpu.is_cpu + and self.cu_seqlens_q_gpu.is_cuda + and self.indices.is_cuda + and self.last_page_len_cpu.is_cpu + and self.seq_lens_cpu.is_cpu + ) + + def get_positions(self) -> torch.Tensor: + return self.positions + + def get_last_indices(self, bs: int) -> torch.Tensor: + return self.cu_seqlens_q_gpu[1 : 1 + bs] - 1 + + +class FlashInferBackend(BaseAttnBackend): + def __init__( + self, + config: ModelConfig, + kvcache: BaseKVCache, + page_table: torch.Tensor, + ) -> None: + from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + BatchPrefillWithPagedKVCacheWrapper, + ) + + self.config = config + self.kvcache = kvcache + self.device = kvcache.device + self.float_workspace_buffer = torch.empty( + 128 * 1024 * 1024, dtype=torch.uint8, device=self.device + ) + self.prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( + self.float_workspace_buffer, + kv_layout="NHD", + backend="fa2", # flashinfer fa3 is buggy, use fa2 instead + ) + self.decode_wrappers = BatchDecodeWithPagedKVCacheWrapper( + self.float_workspace_buffer, + kv_layout="NHD", + ) + + # NOTE: some hack to reuse the int_workspace_buffer + self.int_workspace_buffer = self.prefill_wrapper._int_workspace_buffer + self.decode_wrappers._int_workspace_buffer = self.int_workspace_buffer + + # initialize some data members + tp_size = get_tp_info().size + self.qo_head_local = divide_even(self.config.num_qo_heads, tp_size) + self.kv_head_local = divide_even(self.config.num_kv_heads, tp_size) + + self.cached_ones_cpu: torch.Tensor = torch.tensor([], dtype=torch.int32, pin_memory=True) + # for cuda graph + self.capture_bs: List[int] = [] + self.max_graph_bs = 0 + self.graph_wrappers: Dict[int, CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = {} + self.capture: FICaptureData | None = None + self.page_table = page_table + + @staticmethod + def _initialize_metadata_once(metadata: FIMetadata) -> None: + if metadata.initialized: + return + + from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + CUDAGraphBatchDecodeWithPagedKVCacheWrapper, + ) + + metadata.initialized = True + + is_cuda_graph = isinstance(metadata.wrapper, CUDAGraphBatchDecodeWithPagedKVCacheWrapper) + is_decode = isinstance(metadata.wrapper, BatchDecodeWithPagedKVCacheWrapper) or is_cuda_graph + + if is_cuda_graph and logger.isEnabledFor(logging.DEBUG): + global _fi_debug_counter + if _fi_debug_counter <= 5: + logger.debug("plan() for CUDAGraphWrapper: indices shape=%s", metadata.indices.shape) + logger.debug(" cu_seqlens_k_cpu=%s", metadata.cu_seqlens_k_cpu.tolist()) + + if is_decode: + # Decode wrapper (both regular and CUDA graph variants) + # sm_scale must be passed to plan() for decode, not run() + metadata.wrapper.plan( + indptr=metadata.cu_seqlens_k_cpu, + indices=metadata.indices, + last_page_len=metadata.last_page_len_cpu, + num_qo_heads=metadata.num_qo_heads, + num_kv_heads=metadata.num_kv_heads, + head_dim=metadata.head_dim, + page_size=metadata.page_size, + pos_encoding_mode=metadata.pos_encoding_mode, + seq_lens=metadata.seq_lens_cpu, + data_type=metadata.dtype, + q_data_type=metadata.dtype, + kv_data_type=metadata.dtype, + sm_scale=metadata.sm_scale, + non_blocking=True, + ) + else: + # Prefill wrapper - sm_scale passed to plan() + metadata.wrapper.plan( + qo_indptr=metadata.cu_seqlens_q_cpu, + paged_kv_indptr=metadata.cu_seqlens_k_cpu, + paged_kv_indices=metadata.indices, + paged_kv_last_page_len=metadata.last_page_len_cpu, + num_qo_heads=metadata.num_qo_heads, + num_kv_heads=metadata.num_kv_heads, + head_dim_qk=metadata.head_dim, + page_size=metadata.page_size, + pos_encoding_mode=metadata.pos_encoding_mode, + seq_lens=metadata.seq_lens_cpu, + q_data_type=metadata.dtype, + kv_data_type=metadata.dtype, + sm_scale=metadata.sm_scale, + non_blocking=True, + causal=True, + ) + + def _get_ones_cpu(self, bs: int) -> torch.Tensor: + if bs <= len(self.cached_ones_cpu): + return self.cached_ones_cpu[:bs] + # padding to next pow of 2 + next_len = _next_power_of_2(bs) + self.cached_ones_cpu = torch.ones(next_len, dtype=torch.int32, pin_memory=True) + return self.cached_ones_cpu[:bs] + + def forward( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, layer_id: int, batch: TTSBatch + ) -> torch.Tensor: + metadata = batch.attn_metadata + assert isinstance(metadata, FIMetadata) + self._initialize_metadata_once(metadata) + self.kvcache.store_kv(k, v, batch.out_loc, layer_id) + kv_cache = (self.kvcache.k_cache(layer_id), self.kvcache.v_cache(layer_id)) + + # Debug: check attention inputs/outputs for first few steps at layer 0 + global _fi_debug_counter + is_capturing = torch.cuda.is_current_stream_capturing() + if (logger.isEnabledFor(logging.DEBUG) + and layer_id == 0 and batch.is_decode and not is_capturing + and _fi_debug_counter < 10): + _fi_debug_counter += 1 + step = _fi_debug_counter + logger.debug("ATTN Step %d Layer 0", step) + logger.debug(" Q shape: %s, norm: %.4f, mean: %.6f", q.shape, q.norm().item(), q.mean().item()) + logger.debug(" K shape: %s, norm: %.4f, mean: %.6f", k.shape, k.norm().item(), k.mean().item()) + logger.debug(" V shape: %s, norm: %.4f, mean: %.6f", v.shape, v.norm().item(), v.mean().item()) + logger.debug(" out_loc: %s", batch.out_loc.tolist()) + logger.debug(" positions: %s", metadata.positions.tolist()) + logger.debug(" indices shape: %s, first 10: %s", metadata.indices.shape, metadata.indices[:10].tolist()) + logger.debug(" cu_seqlens_k: %s", metadata.cu_seqlens_k_cpu.tolist()) + k_cache, v_cache = kv_cache + if batch.out_loc.numel() > 0: + slot = batch.out_loc[0].item() + logger.debug(" KV cache slot %d: k_norm=%.4f", slot, k_cache.view(-1, k_cache.shape[-1])[slot].norm().item()) + + output = metadata.wrapper.run(q=q, paged_kv_cache=kv_cache) + + if (logger.isEnabledFor(logging.DEBUG) + and layer_id == 0 and batch.is_decode and not is_capturing + and _fi_debug_counter <= 10): + logger.debug(" Output shape: %s, norm: %.4f, mean: %.6f", output.shape, output.norm().item(), output.mean().item()) + + return output + + def prepare_metadata(self, batch: TTSBatch) -> None: + reqs = batch.padded_reqs + + padded_size = len(reqs) + seqlens_q = [req.extend_len for req in reqs] + seqlens_k = [req.device_len for req in reqs] + cached_lens = [req.cached_len for req in reqs] + max_seqlen_q = max(seqlens_q) + cpu_kwargs = {"device": "cpu", "dtype": torch.int32, "pin_memory": True} + + device = self.device + seq_len_cpu = torch.tensor(seqlens_k, **cpu_kwargs) + cu_seqlens_k_cpu = torch.tensor([0] + seqlens_k, **cpu_kwargs).cumsum_(dim=0) + if max_seqlen_q == 1: # decode with all extend_len = 1 + cu_seqlens_q_cpu = torch.arange(0, padded_size + 1, **cpu_kwargs) + elif all(l == 0 for l in cached_lens): # prefill with no cache hit + cu_seqlens_q_cpu = cu_seqlens_k_cpu + else: # normal extend prefill, with partial cache hit + cu_seqlens_q_cpu = torch.tensor([0] + seqlens_q, **cpu_kwargs).cumsum_(dim=0) + batch.attn_metadata = FIMetadata( + positions=make_positions(device, reqs), + cu_seqlens_q_cpu=cu_seqlens_q_cpu, + cu_seqlens_k_cpu=cu_seqlens_k_cpu, + cu_seqlens_q_gpu=cu_seqlens_q_cpu.to(device, non_blocking=True), + indices=torch.cat([self.page_table[req.table_idx, : req.device_len] for req in reqs]), + last_page_len_cpu=self._get_ones_cpu(padded_size), + num_qo_heads=self.qo_head_local, + num_kv_heads=self.kv_head_local, + head_dim=self.config.head_dim, + page_size=1, + pos_encoding_mode="NONE", + seq_lens_cpu=seq_len_cpu, + dtype=self.kvcache.dtype, + wrapper=self.decode_wrappers if batch.is_decode else self.prefill_wrapper, + sm_scale=1.0 / math.sqrt(self.config.head_dim), + ) + + def init_capture_graph( + self, max_seq_len: int, bs_list: List[int], frame_width: int = 1 + ) -> None: + assert self.capture is None, "Capture already initialized." + max_bs = max(bs_list) + capture = FICaptureData.create(max_bs, max_seq_len, self.kvcache.device, frame_width) + capture.page_table = capture.page_table.view(-1) # use 1D as ragged indices + self.max_graph_bs = max_bs + self.capture = capture + self.capture_bs = sorted(bs_list) + + @cached_property + def use_tensor_cores(self) -> bool: + if (overriden_value := ENV.FLASHINFER_USE_TENSOR_CORES.value) is not None: + logger.warning(f"Overriding FlashInfer tensor core usage to {overriden_value}") + return overriden_value + GQA = self.config.num_qo_heads // self.config.num_kv_heads + return GQA >= 4 + + def prepare_for_capture(self, batch: TTSBatch) -> None: + from flashinfer import CUDAGraphBatchDecodeWithPagedKVCacheWrapper + + bs = batch.size + assert bs in self.capture_bs and bs not in self.graph_wrappers and self.capture + batch.padded_reqs = batch.reqs + capture = self.capture + self.graph_wrappers[bs] = CUDAGraphBatchDecodeWithPagedKVCacheWrapper( + self.float_workspace_buffer, + kv_layout="NHD", + use_tensor_cores=self.use_tensor_cores, + indptr_buffer=capture.cu_seqlens_k[: bs + 1], + indices_buffer=capture.indices, + last_page_len_buffer=capture.one_tensor[:bs], + ) + self.graph_wrappers[bs]._int_workspace_buffer = self.int_workspace_buffer + self.prepare_metadata(batch) + metadata = batch.attn_metadata + assert isinstance(metadata, FIMetadata) + # Copy metadata tensors to capture buffers and update metadata to use capture buffers + # This ensures CUDA graph uses fixed memory addresses + indices_len = metadata.indices.shape[0] + capture.indices[:indices_len].copy_(metadata.indices) + capture.cu_seqlens_k[: bs + 1].copy_(metadata.cu_seqlens_k_cpu.to(capture.indices.device)) + capture.seq_lens[:bs].copy_(metadata.seq_lens_cpu.to(capture.indices.device)) + # Update metadata to point to capture buffers (critical for CUDA graph) + metadata.indices = capture.indices[:indices_len] + metadata.wrapper = self.graph_wrappers[bs] + metadata.positions = capture.positions[:bs] + batch.input_ids = capture.input_ids[:bs] + batch.out_loc = capture.out_loc[:bs] + self._initialize_metadata_once(metadata) + + def prepare_for_replay(self, batch: TTSBatch) -> None: + metadata, bs = batch.attn_metadata, batch.padded_size + assert isinstance(metadata, FIMetadata) and not metadata.initialized + assert self.capture is not None and bs in self.capture_bs + capture = self.capture + + if logger.isEnabledFor(logging.DEBUG) and _fi_debug_counter < 5: + logger.debug("prepare_for_replay: bs=%d, indices_len=%d", bs, metadata.indices.shape[0]) + logger.debug(" cu_seqlens_k_cpu=%s", metadata.cu_seqlens_k_cpu.tolist()) + + # Copy all dynamic tensors to capture buffers (fixed memory addresses for CUDA graph) + capture.input_ids[:bs].copy_(batch.input_ids) + capture.out_loc[:bs].copy_(batch.out_loc) + capture.positions[:bs].copy_(metadata.positions) + # Copy indices (page table entries) - critical for KV cache lookups + indices_len = metadata.indices.shape[0] + capture.indices[:indices_len].copy_(metadata.indices) + # Copy cu_seqlens_k for the wrapper's indptr + capture.cu_seqlens_k[: bs + 1].copy_(metadata.cu_seqlens_k_cpu.to(capture.indices.device)) + # Copy seq_lens for the wrapper + capture.seq_lens[:bs].copy_(metadata.seq_lens_cpu.to(capture.indices.device)) + # Update metadata to point to capture buffers (CRITICAL: must match prepare_for_capture) + # This ensures CUDA graph uses fixed memory addresses for all inputs + metadata.indices = capture.indices[:indices_len] + metadata.positions = capture.positions[:bs] + metadata.wrapper = self.graph_wrappers[bs] + # Update batch input_ids and out_loc to point to capture buffers + batch.input_ids = capture.input_ids[:bs] + batch.out_loc = capture.out_loc[:bs] + self._initialize_metadata_once(metadata) diff --git a/zonos2/attention/utils.py b/zonos2/attention/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..10a6bb7b2b199b6228439f11bf246b748e08a9a5 --- /dev/null +++ b/zonos2/attention/utils.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, List + +import torch + +if TYPE_CHECKING: + from zonos2.core import TTSReq + + +@dataclass +class BaseCaptureData: + input_ids: torch.Tensor + seq_lens: torch.Tensor + positions: torch.Tensor + cu_seqlens_k: torch.Tensor + cu_seqlens_q: torch.Tensor + page_table: torch.Tensor + out_loc: torch.Tensor + + @classmethod + def create( + cls, max_bs: int, max_seq_len: int, device: torch.device, frame_width: int = 1, **kwargs + ): + # Audio token frames are 2D (batch, frame_width); frame_width == 1 keeps a 1D layout + if frame_width > 1: + input_ids = torch.zeros((max_bs, frame_width), dtype=torch.int32, device=device) + else: + input_ids = torch.zeros((max_bs,), dtype=torch.int32, device=device) + return cls( + input_ids=input_ids, + seq_lens=torch.ones((max_bs,), dtype=torch.int32, device=device), + positions=torch.zeros((max_bs,), dtype=torch.int32, device=device), + cu_seqlens_k=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device), + cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device), + page_table=torch.zeros((max_bs, max_seq_len), dtype=torch.int32, device=device), + out_loc=torch.zeros((max_bs,), dtype=torch.int32, device=device), + **kwargs, + ) + + +def make_positions(device: torch.device, reqs: List[TTSReq]) -> torch.Tensor: + needed_size = sum(req.extend_len for req in reqs) + indices_host = torch.empty(needed_size, dtype=torch.int32, pin_memory=True) + offset = 0 + for req in reqs: + length = req.extend_len + torch.arange( + req.cached_len, + req.device_len, + dtype=torch.int32, + out=indices_host[offset : offset + length], + ) + offset += length + return indices_host.to(device, non_blocking=True) diff --git a/zonos2/benchmark/client.py b/zonos2/benchmark/client.py new file mode 100644 index 0000000000000000000000000000000000000000..68475e63725bf5df494122c5f65a6831a9ce3508 --- /dev/null +++ b/zonos2/benchmark/client.py @@ -0,0 +1,501 @@ +from __future__ import annotations + +import asyncio +import random +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Any, Dict, List, Tuple, overload + +from openai import AsyncOpenAI as OpenAI +from pydantic import BaseModel +from tqdm.asyncio import tqdm +from zonos2.utils import UNSET, Unset, init_logger + +logger = init_logger(__name__) + + +@dataclass(frozen=True) +class BenchmarkTrace: + timestamp: float + message: str # unit (second) + output_length: int # output length in tokens + input_length: int | None = None # input length in tokens, optional + + +@dataclass(frozen=True) +class BenchOneResult: + tics: List[float] + input_len: int + output_len: int + + def as_json(self) -> List[float]: + return [self.input_len, self.output_len] + self.tics + + @staticmethod + def from_json(raw: List[float]) -> BenchOneResult: + # check raw[0] and raw[1] are integers + assert raw[0].is_integer() and raw[1].is_integer() + return BenchOneResult(tics=raw[2:], input_len=int(raw[0]), output_len=int(raw[1])) + + +@dataclass(frozen=True) +class RawResult: + input_len: int | None + output_len: int + message: str + tics: List[float] + + +@dataclass +class Counter: + current: int = 0 + history_max: int = 0 + + def inc(self, n=1): + self.current += n + self.history_max = max(self.history_max, self.current) + + def dec(self, n=1): + self.current -= n + assert self.current >= 0 + + +@dataclass +class Console: + input_pbar: tqdm + output_pbar: tqdm + prefill_pbar: tqdm + decode_pbar: tqdm + disabled: bool + inflight_counter: Counter = field(default_factory=Counter) + queue_counter: Counter = field(default_factory=Counter) + + def update_input(self, n=1): + self.input_pbar.update(n) + self.input_pbar.refresh() + self.inflight_counter.inc(n) + self.queue_counter.inc(n) + + def update_output(self, n=1): + self.output_pbar.update(n) + self.output_pbar.refresh() + self.inflight_counter.dec(n) + + def update_prefill(self, n=1): + self.prefill_pbar.update(n) + self.prefill_pbar.refresh() + self.queue_counter.dec(n) + + def update_decode(self, n=1): + self.decode_pbar.update(n) + + @contextmanager + def inflight(self, n=1): + self.update_input(n) + yield + self.update_output(n) + + @contextmanager + def log_stats(self): + yield + self.input_pbar.close() + self.output_pbar.close() + self.prefill_pbar.close() + self.decode_pbar.close() + if not self.disabled: + max_inflight = self.inflight_counter.history_max + max_queue = self.queue_counter.history_max + logger.info(f"Max inflight requests: {max_inflight}, Max queued requests: {max_queue}") + + +@dataclass(frozen=True) +class BenchmarkResult: + raw_data: List[BenchOneResult] + + def as_json(self) -> List[List[float]]: + return [r.as_json() for r in self.raw_data] + + @staticmethod + def from_json(raw: List[List[float]]) -> BenchmarkResult: + return BenchmarkResult(raw_data=[BenchOneResult.from_json(r) for r in raw]) + + +def make_console(num_requests: int, sum_output_length: int, use_pbar: bool = True) -> Console: + BAR_FORMAT_0 = ( + "{desc:<10} {percentage:3.0f}%|{bar}|" + " {n_fmt:>5}/{total_fmt} " + "[{rate_fmt:>12} {elapsed:>8}/{remaining:<8}]" + ) + BAR_FORMAT_1 = BAR_FORMAT_0 + n_fmt_align = 5 + prefill_tokens = num_requests + decode_tokens = sum_output_length - prefill_tokens + + if len(str(decode_tokens)) > n_fmt_align: + n_fmt_align = len(str(decode_tokens)) + BAR_FORMAT_0 = BAR_FORMAT_0.replace("{n_fmt:>5}", "{n_fmt:>" + str(n_fmt_align) + "}") + BAR_FORMAT_1 = BAR_FORMAT_0 + + if len(str(prefill_tokens)) < len(str(decode_tokens)): + old_align_str = "{n_fmt:>" + str(n_fmt_align) + "}" + n_fmt_align += len(str(decode_tokens)) - len(str(prefill_tokens)) + BAR_FORMAT_0 = BAR_FORMAT_0.replace(old_align_str, "{n_fmt:>" + str(n_fmt_align) + "}") + + disabled = not use_pbar + input_pbar = tqdm( + total=num_requests, + desc="Requests sent", + position=0, + bar_format=BAR_FORMAT_0, + disable=disabled, + ) + output_pbar = tqdm( + total=num_requests, + desc="Requests done", + position=1, + bar_format=BAR_FORMAT_0, + disable=disabled, + ) + prefill_pbar = tqdm( + total=prefill_tokens, + desc="Prefill token", + position=2, + bar_format=BAR_FORMAT_0, + disable=disabled, + ) + decode_pbar = tqdm( + total=decode_tokens, + desc="Decode token ", + position=3, + bar_format=BAR_FORMAT_1, + disable=disabled, + ) + return Console( + input_pbar=input_pbar, + output_pbar=output_pbar, + prefill_pbar=prefill_pbar, + decode_pbar=decode_pbar, + disabled=disabled, + ) + + +def generate_prompt(tokenizer: Any, n: int) -> str: + """Generate a prompt of approximately `n` tokens using the provided tokenizer.""" + vocab_size = tokenizer.vocab_size // 2 + token_ids = [random.randint(0, vocab_size) for _ in range(n - 1)] + + for _ in range(64): + prompt = tokenizer.decode(token_ids) + token_ids = tokenizer.encode(prompt, add_special_tokens=False) + if len(token_ids) == n: + return prompt + if len(token_ids) < n: + need = n - len(token_ids) + token_ids.extend([random.randint(0, vocab_size) for _ in range(need)]) + else: + token_ids = token_ids[:n] + + raise ValueError("Failed to generate a message of the desired length.") + + +async def benchmark_one( + client: OpenAI, + prompt: str, + output_length: int, + model: str, + *, + pbar: Console | bool = True, + extra_body: Dict[str, Any] | None = None, + input_length: int | None = None, # a hack to force input length +) -> RawResult: + if isinstance(pbar, bool): + pbar = make_console(1, output_length, use_pbar=pbar) + with pbar.inflight(1): + kwargs = { + "ignore_eos": True, + "top_k": 1, + } + # this is an internal kwargs that might work for our system + if input_length is not None: + kwargs["input_length_override"] = input_length + kwargs.update(extra_body or {}) # can override kwargs + response = await client.chat.completions.create( + model=model, + stream=True, + messages=[ + { + "role": "user", + "content": prompt, + }, + ], + max_tokens=output_length, + temperature=0.0, + extra_body=kwargs, + ) + tics = [time.perf_counter()] + async for _ in response: + tics.append(time.perf_counter()) + if len(tics) == 2: + pbar.update_prefill() + elif len(tics) <= output_length + 1: + pbar.update_decode() + return RawResult( + input_len=input_length, + output_len=output_length, + message=prompt, + tics=tics, + ) + + +async def benchmark_one_batch( + client: OpenAI, + prompts: List[str], + output_lengths: List[int] | int, + model: str, + *, + extra_body: Dict[str, Any] | None = None, + input_lengths: List[int | None] | None = None, + pbar: Console | bool = True, +) -> List[RawResult]: + if isinstance(output_lengths, int): + output_lengths = [output_lengths] * len(prompts) + if isinstance(pbar, bool): + pbar = make_console(len(prompts), sum(output_lengths), use_pbar=pbar) + if input_lengths is None: + l: List[int | None] = [None] * len(prompts) + input_lengths = l # work-around for typing bug + + tasks = [ + benchmark_one( + client=client, + prompt=prompt, + output_length=output_length, + model=model, + pbar=pbar, + extra_body=extra_body, + input_length=input_length, + ) + for prompt, output_length, input_length in zip( + prompts, output_lengths, input_lengths, strict=True + ) + ] + with pbar.log_stats(): + return await asyncio.gather(*tasks) + + +async def benchmark_trace( + client: OpenAI, + msgs: List[BenchmarkTrace], + model: str, + *, + pbar: Console | bool = True, +) -> List[RawResult]: + if isinstance(pbar, bool): + sum_output_len = sum(msg.output_length for msg in msgs) + pbar = make_console(len(msgs), sum_output_len, use_pbar=pbar) + start = time.perf_counter() + offset = min(msg.timestamp for msg in msgs) - 1 + + async def benchmark_timed(msg: BenchmarkTrace): + target = start + msg.timestamp - offset + await asyncio.sleep(max(0, target - time.perf_counter())) + return await benchmark_one( + client, msg.message, msg.output_length, model, pbar=pbar, input_length=msg.input_length + ) + + tasks = [benchmark_timed(msg) for msg in msgs] + with pbar.log_stats(): + return await asyncio.gather(*tasks) + + +@overload +def process_benchmark_results(raw_data: List[RawResult], tokenizer: Any) -> BenchmarkResult: ... + + +@overload +def process_benchmark_results(raw_data: List[RawResult]) -> None: ... + + +def process_benchmark_results( + raw_data: List[RawResult], + tokenizer: Any = UNSET, +) -> BenchmarkResult | None: + accum_times: List[float] = [] + first_times: List[float] = [] + results = [r.tics for r in raw_data] + for tics in results: + deltas: List[float] = [] + for i in range(len(tics) - 1): + diff = tics[i + 1] - tics[i] + deltas.append(diff) + first_times.append(deltas[0]) + accum_times.extend(deltas[1:]) + + e2e_times = [tics[-1] - tics[0] for tics in results] + first_times.sort() + accum_times.sort() + e2e_times.sort() + + def _print_stats(times: List[float], scale: float = 1.0) -> Tuple[float, ...]: + assert len(times) > 0 + return ( + scale * sum(times) / len(times), # avg + scale * times[int(len(times) * 0.5)], # p50 + scale * times[int(len(times) * 0.9)], # p90 + scale * times[int(len(times) * 0.99)], # p99 + scale * max(times), # max + ) + + def _fmt(x: float) -> str: + if x >= 1000: + return f"{int(x):>6}" + elif x >= 10: + return f"{x:>6.2f}" + else: + return f"{x:>6.4f}" + + avg_ttft, p50_ttft, p90_ttft, p99_ttft, max_ttft = _print_stats(first_times, 1000) + avg_tpot, p50_tpot, p90_tpot, p99_tpot, max_tpot = _print_stats(accum_times, 1000) + avg_e2e, p50_e2e, p90_e2e, p99_e2e, max_e2e = _print_stats(e2e_times) + + min_time = min(min(r) for r in results) + max_time = max(max(r) for r in results) + dur = max_time - min_time + assert dur > 0, "Duration must be positive" + + num_tokens = sum(len(tic) for tic in results) + num_requests = len(results) + + logger.info(f"Num requests: #{num_requests}, Num tokens: #{num_tokens}") + logger.info( + f"TTFT: {_fmt(avg_ttft)} ms (p50: {_fmt(p50_ttft)} ms, p90: {_fmt(p90_ttft)} ms," + f" p99: {_fmt(p99_ttft)} ms, max: {_fmt(max_ttft)} ms)" + ) + logger.info( + f"TPOT: {_fmt(avg_tpot)} ms (p50: {_fmt(p50_tpot)} ms, p90: {_fmt(p90_tpot)} ms," + f" p99: {_fmt(p99_tpot)} ms, max: {_fmt(max_tpot)} ms)" + ) + logger.info( + f"E2E: {_fmt(avg_e2e) } s (p50: {_fmt(p50_e2e) } s, p90: {_fmt(p90_e2e) } s," + f" p99: {_fmt(p99_e2e) } s, max: {_fmt(max_e2e) } s)" + ) + logger.info(f"Duration: {_fmt(dur)} s") + logger.info(f"Throughput: {_fmt(num_tokens / dur)} token/s, {_fmt(num_requests / dur)} req/s") + + # normalize the time to start from zero + results = [[r - min_time for r in tics] for tics in results] + if isinstance(tokenizer, Unset): + return None + + return BenchmarkResult( + raw_data=[ + BenchOneResult( + tics=r.tics, + input_len=( + r.input_len + if r.input_len is not None + else len(tokenizer.encode(r.message, add_special_tokens=False)) + ), + output_len=r.output_len, + ) + for r in raw_data + ] + ) + + +def read_qwen_trace( + file_path: str, + tokenizer: Any, + n: int | None = None, + dummy: bool = False, +) -> List[BenchmarkTrace]: + class JSONInput(BaseModel): + chat_id: int + parent_chat_id: int + timestamp: float + input_length: int + output_length: int + type: str # unused + turn: int # unused + hash_ids: List[int] # unused + + with open(file_path, "r") as f: + lines = f.readlines() + if n is not None: + lines = lines[:n] + objs = [JSONInput.model_validate_json(line) for line in lines] + if dummy: + prompt = generate_prompt(tokenizer, max(obj.input_length for obj in objs)) + ids = tokenizer.encode(prompt, add_special_tokens=False) + _get_prompt = lambda obj: tokenizer.decode(ids[: obj.input_length]) + else: + _get_prompt = lambda obj: generate_prompt(tokenizer, obj.input_length) + return [ + BenchmarkTrace( + timestamp=obj.timestamp, + message=_get_prompt(obj), + input_length=obj.input_length, + output_length=obj.output_length, + ) + for obj in objs + ] + + +def read_mooncake_trace( + file_path: str, + tokenizer: Any, + n: int | None = None, + dummy: bool = False, +) -> List[BenchmarkTrace]: + class JSONInput(BaseModel): + timestamp: int + input_length: int + output_length: int + hash_ids: List[int] # unused for now + + with open(file_path, "r") as f: + lines = f.readlines() + if n is not None: + lines = lines[:n] + objs = [JSONInput.model_validate_json(line) for line in lines] + if dummy: + prompt = generate_prompt(tokenizer, max(obj.input_length for obj in objs)) + ids = tokenizer.encode(prompt, add_special_tokens=False) + _get_prompt = lambda obj: tokenizer.decode(ids[: obj.input_length]) + else: + _get_prompt = lambda obj: generate_prompt(tokenizer, obj.input_length) + return [ + BenchmarkTrace( + timestamp=obj.timestamp / 1000, + message=_get_prompt(obj), + input_length=obj.input_length, + output_length=obj.output_length, + ) + for obj in objs + ] + + +def scale_traces( + traces: List[BenchmarkTrace], + scale: float, +) -> List[BenchmarkTrace]: + min_tic = min(trace.timestamp for trace in traces) + return sorted( + [ + BenchmarkTrace( + timestamp=(trace.timestamp - min_tic) * scale, + message=trace.message, + input_length=trace.input_length, + output_length=trace.output_length, + ) + for trace in traces + ], + key=lambda x: x.timestamp, + ) + + +async def get_model_name(client: OpenAI) -> str: + async for model in client.models.list(): + return model.id + raise ValueError("No models available") diff --git a/zonos2/benchmark/perf.py b/zonos2/benchmark/perf.py new file mode 100644 index 0000000000000000000000000000000000000000..ec6968a6aba60ce938e934dc11470d825f618e5e --- /dev/null +++ b/zonos2/benchmark/perf.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any, Callable, Dict, Tuple + +from zonos2.utils import init_logger + +logger = init_logger(__name__) + + +def perf_cuda( + f: Callable[[], Any], + *, + init_stream: bool = True, + repetitions: int = 10, + cuda_graph_repetitions: int | None = 10, +) -> float: + import torch + + assert repetitions > 0 + tic = torch.cuda.Event(enable_timing=True) + toc = torch.cuda.Event(enable_timing=True) + stream = torch.cuda.Stream() + torch.cuda.synchronize() + if init_stream: + stream = torch.cuda.Stream() + else: + stream = torch.cuda.current_stream() + + with torch.cuda.stream(stream): + f() + if N := cuda_graph_repetitions: + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for _ in range(N): + f() + replay = g.replay + del g + else: + replay = f + N = 1 + + torch.cuda.synchronize() + + replay() + tic.record() + for _ in range(repetitions): + replay() + toc.record() + toc.synchronize() + dur = tic.elapsed_time(toc) + return dur / (N * repetitions) + + +def compare_memory_kernel_perf( + *, + baseline: Callable[[], Any], + our_impl: Callable[[], Any], + memory_footprint: int, # in bytes + description: str = " ", + extra_kwargs: Dict[str, Any] | None = None, + need_latency: bool = True, +) -> Tuple[float, float]: + extra_kwargs = extra_kwargs or {} + + dur = perf_cuda(baseline, **extra_kwargs) + bandwidth_0 = memory_footprint / (dur * 1e6) # GB/s + latency_msg = f"{dur:8.3f} ms | " if need_latency else "" + message_0 = f"Baseline: {latency_msg}{bandwidth_0:8.3f} GB/s" + + dur = perf_cuda(our_impl, **extra_kwargs) + bandwidth_1 = memory_footprint / (dur * 1e6) # GB/s + latency_msg = f"{dur:8.3f} ms | " if need_latency else "" + logger.info(f"{description}{message_0} | Our Impl: {latency_msg}{bandwidth_1:8.3f} GB/s") + return bandwidth_0, bandwidth_1 diff --git a/zonos2/core.py b/zonos2/core.py new file mode 100644 index 0000000000000000000000000000000000000000..b6d1596072ea2be970f0f61d6505d81c492162f4 --- /dev/null +++ b/zonos2/core.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List, Literal + +import torch + +if TYPE_CHECKING: + from zonos2.attention import BaseAttnBackend, BaseAttnMetadata + from zonos2.kvcache import BaseCacheHandle + + +@dataclass +class Context: + page_size: int + attn_backend: BaseAttnBackend + _batch: TTSBatch | None = field(default=None, init=False) + + @property + def batch(self) -> TTSBatch: + assert self._batch is not None, "No active batch in context" + return self._batch + + @contextmanager + def forward_batch(self, batch: TTSBatch): + assert self._batch is None, "Nested forward_batch is not allowed" + try: + self._batch = batch + yield + finally: + self._batch = None + + +_GLOBAL_CTX: Context | None = None + + +def set_global_ctx(ctx: Context): + global _GLOBAL_CTX + assert _GLOBAL_CTX is None, "Global context is already set" + _GLOBAL_CTX = ctx + + +def get_global_ctx() -> Context: + assert _GLOBAL_CTX is not None, "Global context is not set" + return _GLOBAL_CTX + + +# ============================================================================= +# TTS-specific data structures +# ============================================================================= + + +@dataclass +class TTSSamplingParams: + """Sampling parameters for TTS generation.""" + + temperature: float = 1.15 + topk: int = 106 + top_p: float = 0.0 + min_p: float = 0.18 + max_tokens: int = 1024 + ignore_eos: bool = False + repetition_window: int = 50 + repetition_penalty: float = 1.2 + repetition_codebooks: int = 8 + seed: int | None = None + + +@dataclass(eq=False) +class TTSReq: + """Request class for TTS generation with 2D token format. + + Tokens are in unpacked format: [cb0, cb1, ..., cb8, text_token] per frame. + """ + + input_ids: torch.Tensor # 2D CPU tensor (seq_len, frame_width) + table_idx: int + cached_len: int + output_len: int + uid: int + sampling_params: TTSSamplingParams + cache_handle: BaseCacheHandle + n_codebooks: int = 9 + eoa_id: int = 1024 + eos_frame: int = -1 # Aligned frame where EOS first appeared (-1 = not seen) + eos_countdown: int = -1 # Steps remaining after EOS (-1 = not in countdown) + total_generated: int = 0 # Total frames generated (for logging) + rng: torch.Generator | None = None # Per-request RNG for deterministic sampling + speaker_embedding: torch.Tensor | None = None # 1D CPU float32 tensor + speaker_token_position: int = -1 # Injection position within the prompt sequence + + def __post_init__(self) -> None: + assert self.input_ids.is_cpu + assert self.input_ids.dim() == 2, "TTS input_ids must be 2D (seq_len, frame_width)" + self.device_len = len(self.input_ids) + self.max_device_len = len(self.input_ids) + self.output_len + assert 0 <= self.cached_len < self.device_len <= self.max_device_len + + if self.speaker_embedding is not None: + emb = self.speaker_embedding + if emb.dim() == 2 and emb.shape[0] == 1: + emb = emb.squeeze(0) + if emb.dim() != 1: + raise ValueError( + f"speaker_embedding must be 1D or (1, D), got shape {tuple(emb.shape)}" + ) + self.speaker_embedding = emb.to(dtype=torch.float32, device="cpu") + + if self.speaker_token_position < 0: + # Training convention: reserved speaker slot is at prompt position 0. + self.speaker_token_position = 0 + if self.speaker_token_position >= self.device_len: + self.speaker_token_position = 0 + + @property + def frame_width(self) -> int: + """Number of elements per frame (n_codebooks + extras).""" + return self.input_ids.shape[-1] + + @property + def remain_len(self) -> int: + return self.max_device_len - self.device_len + + @property + def extend_len(self) -> int: + return self.device_len - self.cached_len + + @property + def num_completion_tokens(self) -> int: + """Number of generated tokens (frames).""" + return self.device_len - self.cached_len + + def complete_one(self) -> None: + self.cached_len = self.device_len + self.device_len += 1 + self.total_generated += 1 + + def append_host(self, next_token: torch.Tensor) -> None: + """Append a single frame (unpacked token) to input_ids.""" + assert next_token.dim() == 1, "next_token must be 1D (frame_width,)" + self.input_ids = torch.cat([self.input_ids, next_token.unsqueeze(0)], dim=0) + + def can_decode(self) -> bool: + return self.remain_len > 0 and self.eos_countdown != 0 + + def check_eos(self, audio_codes: List[int]) -> bool: + """Check for EOS and update countdown state. + + Args: + audio_codes: List of audio codebook values for one frame + + Returns: + True if sequence is finished (countdown reached 0) + """ + if self.sampling_params.ignore_eos: + return False + + # Match Zonos2 reference inference: any sampled EOA codebook starts the + # delayed stop countdown. The aligned frame is shifted back by the + # highest EOA codebook index and clamped at zero. + # Use total_generated because this request only sees one decode frame at a time. + if self.eos_frame < 0: + step = self.total_generated - 1 + eos_cols = [c == self.eoa_id for c in audio_codes[: self.n_codebooks]] + if any(eos_cols): + # First EOS: compute aligned frame + max_eos_cb = max(i for i, is_eos in enumerate(eos_cols) if is_eos) + self.eos_frame = max(0, step - max_eos_cb) + self.eos_countdown = self.n_codebooks + 1 + + # Decrement countdown + if self.eos_countdown > 0: + self.eos_countdown -= 1 + if self.eos_countdown == 0: + return True + + return False + + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(table_idx={self.table_idx}, " + f"cached_len={self.cached_len}, device_len={self.device_len}, " + f"max_device_len={self.max_device_len}, eos_frame={self.eos_frame})" + ) + + +@dataclass +class TTSBatch: + """Batch of TTS requests with 2D token format.""" + + reqs: List[TTSReq] + phase: Literal["prefill", "decode"] + # these fields should be set by scheduler + input_ids: torch.Tensor = field(init=False) # (total_tokens, frame_width) + out_loc: torch.Tensor = field(init=False) + padded_reqs: List[TTSReq] = field(init=False) + # this field should be set by attention backend + attn_metadata: BaseAttnMetadata = field(init=False) + # Optional per-batch speaker conditioning data (set by TTS scheduler). + speaker_emb_values: torch.Tensor | None = field(default=None, init=False) + speaker_token_positions: torch.Tensor | None = field(default=None, init=False) + + @property + def is_prefill(self) -> bool: + return self.phase == "prefill" + + @property + def is_decode(self) -> bool: + return self.phase == "decode" + + @property + def size(self) -> int: + return len(self.reqs) + + @property + def padded_size(self) -> int: + return len(self.padded_reqs) diff --git a/zonos2/distributed/__init__.py b/zonos2/distributed/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3c4d843afca2abd585fa1408884d3e718535e7 --- /dev/null +++ b/zonos2/distributed/__init__.py @@ -0,0 +1,12 @@ +from .impl import DistributedCommunicator, destroy_distributed, enable_pynccl_distributed +from .info import DistributedInfo, get_tp_info, set_tp_info, try_get_tp_info + +__all__ = [ + "DistributedInfo", + "get_tp_info", + "set_tp_info", + "enable_pynccl_distributed", + "DistributedCommunicator", + "try_get_tp_info", + "destroy_distributed", +] diff --git a/zonos2/distributed/impl.py b/zonos2/distributed/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..69d269a8be1eaf04bb40fad959b98d2c8feaa0af --- /dev/null +++ b/zonos2/distributed/impl.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, List + +import torch +import torch.distributed as dist + +if TYPE_CHECKING: + from zonos2.distributed import DistributedInfo + from zonos2.kernel import PyNCCLCommunicator + + +@dataclass +class DistributedImpl(ABC): + @abstractmethod + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: ... + + @abstractmethod + def all_gather(self, x: torch.Tensor) -> torch.Tensor: ... + + +@dataclass +class TorchDistributedImpl(DistributedImpl): + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + tp_size = dist.get_world_size() + if tp_size == 1: + return x + dist.all_reduce(x, op=dist.ReduceOp.SUM) + return x + + def all_gather(self, x: torch.Tensor) -> torch.Tensor: + tp_size = dist.get_world_size() + if tp_size == 1: + return x + shape = list(x.shape) + shape[0] = shape[0] * tp_size + out = torch.empty(shape, dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(out, x) + return out + + +@dataclass +class PyNCCLDistributedImpl(DistributedImpl): + comm: PyNCCLCommunicator + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + self.comm.all_reduce(x, "sum") + return x + + def all_gather(self, x: torch.Tensor) -> torch.Tensor: + from .info import get_tp_info + + world_size = get_tp_info().size + output_shape = list(x.shape) + output_shape[0] *= world_size + result = x.new_empty(output_shape) + self.comm.all_gather(result, x) + return result + + +class DistributedCommunicator: + plugins: List[DistributedImpl] = [TorchDistributedImpl()] + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + return self.plugins[-1].all_reduce(x) + + def all_gather(self, x: torch.Tensor) -> torch.Tensor: + return self.plugins[-1].all_gather(x) + + +def enable_pynccl_distributed( + tp_info: DistributedInfo, tp_cpu_group: torch.distributed.ProcessGroup, max_bytes: int +) -> None: + """ + Enable PyNCCL-based distributed communication for tensor parallelism. + """ + if tp_info.size == 1: + return + from zonos2.kernel import init_pynccl + + comm = init_pynccl( + tp_rank=tp_info.rank, + tp_size=tp_info.size, + tp_cpu_group=tp_cpu_group, + max_size_bytes=max_bytes, + ) + + DistributedCommunicator.plugins.append(PyNCCLDistributedImpl(comm)) + + +def destroy_distributed() -> None: + """ + Destroy all the distributed communication plugins. + """ + DistributedCommunicator.plugins = [] diff --git a/zonos2/distributed/info.py b/zonos2/distributed/info.py new file mode 100644 index 0000000000000000000000000000000000000000..c7f72f9949df47c0757f41617cccdee9c73c044e --- /dev/null +++ b/zonos2/distributed/info.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DistributedInfo: # should not export from here + rank: int + size: int + + def __post_init__(self): + assert 0 <= self.rank < self.size + + def is_primary(self) -> bool: + return self.rank == 0 + + +_TP_INFO: DistributedInfo | None = None + + +def set_tp_info(rank: int, size: int) -> None: + global _TP_INFO + if _TP_INFO is not None: + raise RuntimeError("TP info has been set") + _TP_INFO = DistributedInfo(rank, size) + + +def get_tp_info() -> DistributedInfo: + if _TP_INFO is None: + raise RuntimeError("TP info has not been set") + return _TP_INFO + + +def try_get_tp_info() -> DistributedInfo | None: + return _TP_INFO + + +__all__ = ["DistributedInfo", "set_tp_info", "get_tp_info", "try_get_tp_info"] diff --git a/zonos2/engine/__init__.py b/zonos2/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb2cdfef3cb8ed210ab6eace7dfb204b7569f49 --- /dev/null +++ b/zonos2/engine/__init__.py @@ -0,0 +1,4 @@ +from .config import EngineConfig +from .engine import Engine + +__all__ = ["Engine", "EngineConfig"] diff --git a/zonos2/engine/config.py b/zonos2/engine/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4b62d558f2806db134e37322b4c985ab12d04158 --- /dev/null +++ b/zonos2/engine/config.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import cached_property +from typing import TYPE_CHECKING, List + +import torch +from zonos2.distributed import DistributedInfo +from zonos2.utils import cached_load_checkpoint_config + +if TYPE_CHECKING: + from zonos2.models import ModelConfig + + +@dataclass(frozen=True) +class EngineConfig: + model_path: str + tp_info: DistributedInfo + dtype: torch.dtype + max_running_req: int = 256 + attention_backend: str = "auto" + moe_backend: str = "fused_moe" + cuda_graph_bs: List[int] | None = None + cuda_graph_max_bs: int | None = None + page_size: int = 1 + memory_ratio: float = 0.9 + distributed_timeout: float = 60.0 + use_dummy_weight: bool = False + use_pynccl: bool = True + max_seq_len_override: int | None = None + num_page_override: int | None = None # if not None, will override the number of pages + + @cached_property + def checkpoint_config(self): + return cached_load_checkpoint_config(self.model_path) + + @cached_property + def model_config(self) -> ModelConfig: + from zonos2.models import ModelConfig + + return ModelConfig.from_checkpoint_config(self.checkpoint_config) + + @property + def max_seq_len(self) -> int: + if self.max_seq_len_override is not None: + return self.max_seq_len_override + return self.model_config.rotary_config.max_position + + @property + def max_forward_len(self) -> int: + return self.max_seq_len + + @property + def distributed_addr(self) -> str: + return "tcp://127.0.0.1:23333" diff --git a/zonos2/engine/engine.py b/zonos2/engine/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4d0ba60cbfbf8372631c8ea87e981bc2f3dd31 --- /dev/null +++ b/zonos2/engine/engine.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import logging +from datetime import timedelta +from typing import Dict, Tuple + +import torch +from zonos2.attention import create_attention_backend +from zonos2.core import ( + Context, + TTSBatch, + TTSReq, + TTSSamplingParams, + set_global_ctx, +) +from zonos2.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info +from zonos2.kvcache import create_kvcache +from zonos2.layers import set_rope_device +from zonos2.models import create_model, load_checkpoint_weight +from zonos2.utils import divide_even, init_logger, torch_dtype + +from .config import EngineConfig +from .graph import GraphRunner, get_free_memory, mem_GB + +logger = init_logger(__name__) + + +def create_page_table(shape: Tuple[int, int], device: torch.device) -> torch.Tensor: + return torch.zeros(shape, dtype=torch.int32, device=device) + + +def _align_up_32(num: int) -> int: + return (num + 31) // 32 * 32 + + +def _get_frame_width(model_config) -> int: + """Get frame width for multi-codebook models. + + Frame width = n_codebooks + (1 if text_vocab). + """ + width = model_config.n_codebooks + if model_config.text_vocab is not None: + width += 1 + return width + + +class Engine: + def __init__(self, config: EngineConfig): + self.model_config = config.model_config + set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size) + + self.device = torch.device("cuda") + # Importing the model eagerly pulls in sgl_kernel, which creates a CUDA + # context on the current device. The scheduler entry point binds this + # process to its rank's GPU before that import, so an existing context is + # expected; only one on the wrong device indicates a real ordering bug. + if config.tp_info.size > 1 and ( + torch.cuda.is_initialized() + and torch.cuda.current_device() != config.tp_info.rank + ): + raise RuntimeError( + f"CUDA was initialized on cuda:{torch.cuda.current_device()} before " + f"the engine could bind TP rank {config.tp_info.rank}; " + "set the device before importing model code." + ) + if config.tp_info.size > 1: + torch.cuda.set_device(config.tp_info.rank) + self.stream = torch.cuda.Stream() + torch.cuda.set_stream(self.stream) + self.dtype = config.dtype + + self.tp_cpu_group = self._init_communication(config) + init_free_memory = self._sync_get_memory()[1] + logger.info_rank0(f"Free memory before loading model: {mem_GB(init_free_memory)}") + + # load model and determine number of pages + set_rope_device(self.device) + with torch.device("meta"), torch_dtype(config.dtype): + self.model = create_model(config) + + # Log expected shapes before loading (for debugging dimension mismatches) + logger.info_rank0(f"Model config: num_kv_heads={config.model_config.num_kv_heads}, head_dim={config.model_config.head_dim}") + expected_kv_dim = config.model_config.num_kv_heads * config.model_config.head_dim + logger.info_rank0(f"Expected kv_dim per TP rank: {expected_kv_dim}") + + state_dict = self._load_weight_state_dict(config) + self._check_speaker_lda_weights(state_dict) + + # Debug: Check for temp keys in state_dict + if logger.isEnabledFor(logging.DEBUG): + temp_keys = [k for k in state_dict.keys() if '.temp' in k] + logger.debug("Found %d temp keys in state_dict: %s", len(temp_keys), temp_keys[:5]) + if temp_keys: + first_temp = state_dict[temp_keys[0]] + logger.debug( + "First temp tensor: shape=%s, dtype=%s, values=%s", + first_temp.shape, first_temp.dtype, first_temp.flatten()[:4].tolist(), + ) + + # Validate wkv weight shapes match config (catch n_kv_heads mismatch early) + for key, tensor in state_dict.items(): + if ".wkv." in key and "weight" in key: + # wkv.weight should be (2, kv_dim, hidden_size) or (2*kv_dim, hidden_size) + if tensor.dim() == 3: + actual_kv_dim = tensor.shape[1] + else: + actual_kv_dim = tensor.shape[0] // 2 + if actual_kv_dim != expected_kv_dim: + inferred_kv_heads = actual_kv_dim // config.model_config.head_dim + raise ValueError( + f"KV dimension mismatch for {key}: checkpoint has kv_dim={actual_kv_dim} " + f"(implying n_kv_heads={inferred_kv_heads}), but config expects " + f"kv_dim={expected_kv_dim} (n_kv_heads={config.model_config.num_kv_heads}). " + f"Please ensure your checkpoint's params.json has the correct n_kv_heads value " + f"matching the trained model weights." + ) + break # Only need to check one layer + + self.model.load_state_dict(state_dict) + + # Debug: print attention temp values to verify they loaded correctly + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Model type: %s", type(self.model).__name__) + logger.debug("Model has layers: %s", hasattr(self.model, 'layers')) + if hasattr(self.model, 'layers'): + logger.debug("layers type: %s", type(self.model.layers)) + if hasattr(self.model.layers, 'op_list'): + logger.debug("num layers: %d", len(self.model.layers.op_list)) + for i, layer in enumerate(self.model.layers.op_list[:2]): + logger.debug("Layer %d type: %s", i, type(layer).__name__) + if hasattr(layer, 'attention'): + attn = layer.attention + logger.debug("Layer %d attention type: %s", i, type(attn).__name__) + logger.debug("Layer %d has_qk_norm: %s", i, getattr(attn, 'has_qk_norm', 'N/A')) + if hasattr(attn, 'temp'): + temp = attn.temp + if temp is not None: + logger.debug("Layer %d attention.temp: shape=%s, values=%s", i, temp.shape, temp.flatten()[:8].tolist()) + else: + logger.debug("Layer %d attention.temp is None", i) + + self.num_pages = self.dummy_page = self._determine_num_pages(init_free_memory, config) + self.kv_cache = create_kvcache( + model_config=config.model_config, + num_pages=self.num_pages + 1, # +1 for dummy page + device=self.device, + dtype=self.dtype, + ) + # NOTE: make page table 128 aligned (32 * sizeof(int32) == 128 bytes) + self.max_seq_len = _align_up_32(min(config.max_seq_len, self.num_pages)) + self.page_table = create_page_table( # + 1 for dummy request + (config.max_running_req + 1, self.max_seq_len), + device=self.device, + ) + self.attn_backend = create_attention_backend( + config.attention_backend, + config.model_config, + self.kv_cache, + self.page_table, + ) + self.ctx = Context(page_size=1, attn_backend=self.attn_backend) + set_global_ctx(self.ctx) + + post_free_memory = self._sync_get_memory()[0] + logger.info_rank0(f"Free memory after initialization: {mem_GB(post_free_memory)}") + + frame_width = _get_frame_width(self.model_config) + self.dummy_req = TTSReq( + input_ids=torch.zeros((1, frame_width), dtype=torch.int32, device="cpu"), + table_idx=config.max_running_req, + cached_len=0, + output_len=1, + uid=-1, + sampling_params=TTSSamplingParams(), + cache_handle=None, # type: ignore + n_codebooks=self.model_config.n_codebooks, + eoa_id=self.model_config.eoa_id, + ) + self.page_table[self.dummy_req.table_idx].fill_(self.dummy_page) + + self.graph_runner = GraphRunner( + stream=self.stream, + device=self.device, + model=self.model, + attn_backend=self.attn_backend, + cuda_graph_bs=config.cuda_graph_bs, + cuda_graph_max_bs=config.cuda_graph_max_bs, + free_memory=init_free_memory, + max_seq_len=self.max_seq_len, + vocab_size=self.model_config.codebook_size + 2, + dummy_req=self.dummy_req, + ) + + def _init_communication(self, config: EngineConfig) -> torch.distributed.ProcessGroup: + if config.tp_info.size == 1 or config.use_pynccl: + torch.distributed.init_process_group( + backend="gloo", + rank=config.tp_info.rank, + world_size=config.tp_info.size, + timeout=timedelta(seconds=config.distributed_timeout), + init_method=config.distributed_addr, + ) + tp_cpu_group = torch.distributed.group.WORLD + assert tp_cpu_group is not None + max_bytes = ( + config.max_forward_len * config.model_config.hidden_size * self.dtype.itemsize + ) + enable_pynccl_distributed(config.tp_info, tp_cpu_group, max_bytes) + else: + torch.distributed.init_process_group( + backend="nccl", + rank=config.tp_info.rank, + world_size=config.tp_info.size, + timeout=timedelta(seconds=config.distributed_timeout), + init_method=config.distributed_addr, + ) + tp_cpu_group = torch.distributed.new_group(backend="gloo") + assert tp_cpu_group is not None + return tp_cpu_group + + def _load_weight_state_dict(self, config: EngineConfig) -> Dict[str, torch.Tensor]: + if config.use_dummy_weight: + return { + k: torch.randn_like(v, device=self.device) + for k, v in self.model.state_dict().items() + } + else: + return { + k: v.to(self.dtype) + for k, v in load_checkpoint_weight(config.model_path, self.device).items() + } + + def _check_speaker_lda_weights(self, state_dict: Dict[str, torch.Tensor]) -> None: + lda_keys = [key for key in state_dict if key.startswith("speaker_lda_projection.")] + if getattr(self.model, "speaker_lda_projection", None) is None: + for key in lda_keys: + state_dict.pop(key) + if lda_keys: + logger.warning_rank0( + "Dropped %d speaker_lda_projection weights because speaker LDA is not enabled.", + len(lda_keys), + ) + return + + missing = { + "speaker_lda_projection.weight", + "speaker_lda_projection.bias", + }.difference(lda_keys) + if missing: + raise ValueError( + "speaker_lda_dim is configured but the checkpoint is missing " + f"{sorted(missing)}. Use a checkpoint with the LDA projection merged in." + ) + + def _determine_num_pages(self, old_free_memory: int, config: EngineConfig) -> int: + new_free_memory = self._sync_get_memory()[1] + cache_per_page = ( + 2 # key + value + * self.model_config.head_dim + * divide_even(self.model_config.num_kv_heads, config.tp_info.size) + * config.page_size + * self.dtype.itemsize + * self.model_config.num_layers + ) + num_pages = config.num_page_override + if num_pages is None: + model_memory = old_free_memory - new_free_memory + available_memory = int(config.memory_ratio * old_free_memory) - model_memory + num_pages = available_memory // cache_per_page + + assert num_pages > 1, "Not enough memory for KV cache, try reducing --num-tokens" + real_kv_size = num_pages * cache_per_page + logger.info(f"Allocating {num_pages} pages for KV cache, K + V = {mem_GB(real_kv_size)}") + return num_pages + + def _sync_get_memory(self) -> Tuple[int, int]: + """Get the min and max free memory across TP ranks.""" + torch.cuda.synchronize(self.device) + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(self.device) + free_memory = get_free_memory(self.device) + free_mem_tensor = torch.tensor([free_memory, -free_memory], device="cpu", dtype=torch.int64) + torch.distributed.all_reduce( + free_mem_tensor, op=torch.distributed.ReduceOp.MIN, group=self.tp_cpu_group + ) + min_free_memory = int(free_mem_tensor[0].item()) + max_free_memory = -int(free_mem_tensor[1].item()) + if max_free_memory - min_free_memory > 2 * 1024 * 1024 * 1024: + logger.error( + f"Memory across TP ranks are imbalanced:" + f" min {mem_GB(min_free_memory)}, max {mem_GB(max_free_memory)}" + ) + raise RuntimeError("Memory across TP ranks are imbalanced") + + return min_free_memory, max_free_memory + + def forward_batch_tts(self, batch: TTSBatch) -> torch.Tensor: + """Forward pass for TTS batch, returning multi-codebook logits. + + Args: + batch: TTS batch with 2D input tokens + + Returns: + Logits tensor of shape (batch_size, n_codebooks, vocab_size) + """ + assert torch.cuda.current_stream() == self.stream + with self.ctx.forward_batch(batch): + # For TTS, the model outputs multi-codebook logits directly + # The zonos2 model has a MultiParallelLMHead that outputs + # (batch, seq, n_codebooks, vocab_size) + if self.graph_runner.can_use_cuda_graph(batch): + logits = self.graph_runner.replay(batch) + else: + logits = self.model.forward() + + # logits shape: (batch, n_codebooks, vocab) for decode + # or (total_tokens, n_codebooks, vocab) for prefill + # We only care about the last token per sequence + if batch.is_decode: + return logits[: batch.size] + else: + # For prefill, get the LAST token logits for each sequence + # logits shape: (total_tokens, n_codebooks, vocab) + # Need to extract logits at the last position of each sequence + last_indices = [] + cumsum = 0 + for req in batch.reqs: + cumsum += req.extend_len # extend_len = device_len - cached_len + last_indices.append(cumsum - 1) + last_indices = torch.tensor(last_indices, device=logits.device) + return logits[last_indices] + + def shutdown(self) -> None: + self.graph_runner.destroy_cuda_graphs() + torch.distributed.destroy_process_group() + destroy_distributed() diff --git a/zonos2/engine/graph.py b/zonos2/engine/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..4c0c70f17fea2445928825675610534caa3b1598 --- /dev/null +++ b/zonos2/engine/graph.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import gc +from typing import TYPE_CHECKING, Dict, List + +import torch +from tqdm import tqdm +from zonos2.core import TTSBatch, TTSReq, get_global_ctx +from zonos2.distributed import get_tp_info +from zonos2.utils import init_logger + +if TYPE_CHECKING: + from zonos2.attention import BaseAttnBackend + from zonos2.models import Zonos2ForCausalLM + +logger = init_logger(__name__) + + +def _determine_cuda_graph_bs( + cuda_graph_bs: List[int] | None, + cuda_graph_max_bs: int | None, + free_memory: int, +) -> List[int]: + if cuda_graph_bs is not None: + return cuda_graph_bs + + free_memory_gb = free_memory / (1 << 30) + if cuda_graph_max_bs is None: + if free_memory_gb > 80: # H200 + cuda_graph_max_bs = 256 + else: + cuda_graph_max_bs = 160 + + if cuda_graph_max_bs < 1: + return [] + + return [1, 2, 4] + list(range(8, cuda_graph_max_bs + 1, 8)) + + +def mem_GB(size: int) -> str: + return f"{size / (1024**3):.2f} GiB" + + +def get_free_memory(device: torch.device) -> int: + return torch.cuda.mem_get_info(device)[0] + + +class GraphRunner: + def __init__( + self, + stream: torch.cuda.Stream, + device: torch.device, + model: Zonos2ForCausalLM, + attn_backend: BaseAttnBackend, + cuda_graph_bs: List[int] | None, + cuda_graph_max_bs: int | None, + free_memory: int, + max_seq_len: int, + vocab_size: int, + dummy_req: TTSReq, + ) -> None: + cuda_graph_bs = _determine_cuda_graph_bs( + cuda_graph_bs=cuda_graph_bs, + cuda_graph_max_bs=cuda_graph_max_bs, + free_memory=free_memory, + ) + self.attn_backend = attn_backend + if not cuda_graph_bs: + logger.info_rank0("CUDA graph is disabled.") + self.max_graph_bs = 0 + self.graph_bs_list = [] + self.dummy_req = dummy_req + self.stream = stream + self.device = device + self.graph_map = {} + return + + self.max_graph_bs = max(cuda_graph_bs) + self.graph_bs_list = sorted(cuda_graph_bs) + self.dummy_req = dummy_req + self.stream = stream + self.device = device + self.graph_map = self._capture_graphs(max_seq_len, vocab_size, model) + + def _capture_graphs(self, max_seq_len: int, vocab_size: int, model: Zonos2ForCausalLM): + graph_map: Dict[int, torch.cuda.CUDAGraph] = {} + if self.max_graph_bs == 0: + logger.info_rank0("CUDA graph is disabled.") + return graph_map + + self.logits = torch.empty( + (self.max_graph_bs, self.dummy_req.n_codebooks, vocab_size), + dtype=torch.float32, + device=self.device, + ) + frame_width = self.dummy_req.frame_width + self.attn_backend.init_capture_graph( + max_seq_len=max_seq_len, bs_list=self.graph_bs_list, frame_width=frame_width + ) + + torch.cuda.synchronize(self.device) + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(self.device) + + logger.info_rank0(f"Start capturing CUDA graphs with sizes: {self.graph_bs_list}") + free_memory = get_free_memory(self.device) + logger.info_rank0(f"Free GPU memory before capturing CUDA graphs: {mem_GB(free_memory)}") + + pbar = tqdm( + sorted(self.graph_bs_list, reverse=True), + desc="Preparing for capturing CUDA graphs...", + unit="batch", + disable=not get_tp_info().is_primary(), # disable for non-primary ranks + ) + pool = None + for bs in pbar: + free_memory = get_free_memory(self.device) + pbar.desc = f"Capturing graphs: bs = {bs:<3} | avail_mem = {mem_GB(free_memory)}" + pbar.refresh() + graph = torch.cuda.CUDAGraph() + batch = TTSBatch(reqs=[self.dummy_req] * bs, phase="decode") + self.attn_backend.prepare_for_capture(batch) + with get_global_ctx().forward_batch(batch): + self.logits[:bs] = model.forward() + with torch.cuda.graph(graph, pool=pool, stream=self.stream): + self.logits[:bs] = model.forward() + if pool is None: + pool = graph.pool() + graph_map[bs] = graph + + free_memory = get_free_memory(self.device) + logger.info_rank0(f"Free GPU memory after capturing CUDA graphs: {mem_GB(free_memory)}") + return graph_map + + def can_use_cuda_graph(self, batch: TTSBatch) -> bool: + return batch.is_decode and batch.size <= self.max_graph_bs + + def replay(self, batch: TTSBatch) -> torch.Tensor: + assert self.can_use_cuda_graph(batch) + g = self.graph_map[batch.padded_size] + self.attn_backend.prepare_for_replay(batch) + g.replay() + return self.logits[: batch.size] + + def pad_batch(self, batch: TTSBatch) -> int: + padded_size = ( # choose the first available batch size + next(bs for bs in self.graph_bs_list if bs >= batch.size) + if self.can_use_cuda_graph(batch) + else batch.size + ) + batch.padded_reqs = batch.reqs + [self.dummy_req] * (padded_size - batch.size) + return batch.padded_size - batch.size + + # NOTE: This must be called before freeing NCCL resources to prevent program hang + def destroy_cuda_graphs(self) -> None: + del self.graph_map + gc.collect() diff --git a/zonos2/engine/sample.py b/zonos2/engine/sample.py new file mode 100644 index 0000000000000000000000000000000000000000..c6777c0028ebc67957a214fd868f27eda02a7f6f --- /dev/null +++ b/zonos2/engine/sample.py @@ -0,0 +1,170 @@ +"""TTS sampler for multi-codebook audio generation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List + +import torch +from zonos2.tts.sampler import sample_tts + +if TYPE_CHECKING: + from zonos2.core import TTSBatch + + +def make_device_tensor( + data: List, dtype: torch.dtype, device: torch.device +) -> torch.Tensor: + """Create a tensor on device from a list.""" + return torch.tensor(data, dtype=dtype, pin_memory=True).to( + device, non_blocking=True + ) + + +@dataclass +class TTSBatchSamplingArgs: + """Sampling arguments for a TTS batch.""" + + temperatures: torch.Tensor + top_ks: torch.Tensor + top_ps: torch.Tensor + min_ps: torch.Tensor + text_vocab: int + repetition_token_ids: torch.Tensor | None = None + repetition_penalties: torch.Tensor | None = None + generators: list[torch.Generator | None] = field(default_factory=list) + + +@dataclass +class TTSSampler: + """Sampler for multi-codebook TTS output.""" + + device: torch.device + n_codebooks: int + codebook_size: int + text_vocab: int + + def prepare( + self, batch: TTSBatch, token_pool: torch.Tensor | None = None + ) -> TTSBatchSamplingArgs: + """Prepare sampling arguments from a TTS batch. + + Args: + batch: TTS batch with requests + token_pool: Optional device-side token pool for recent generated history + + Returns: + TTSBatchSamplingArgs with per-sequence sampling parameters + """ + params = [r.sampling_params for r in batch.reqs] + + MIN_T = 1e-6 + temps = [max(p.temperature, MIN_T) for p in params] + top_ks = [p.topk if p.topk >= 1 else self.codebook_size for p in params] + top_ps = [min(max(p.top_p, 0.0), 1.0) for p in params] + min_ps = [max(p.min_p, 0.0) for p in params] + repetition_windows = [max(int(p.repetition_window), 0) for p in params] + repetition_penalties = [max(float(p.repetition_penalty), 1.0) for p in params] + repetition_codebooks = [ + self.n_codebooks + if int(p.repetition_codebooks) < 0 + else min(max(int(p.repetition_codebooks), 0), self.n_codebooks) + for p in params + ] + generators = [r.rng for r in batch.reqs] + repetition_token_ids = None + repetition_penalties_tensor = None + + active_windows = [ + min(window, r.total_generated, r.device_len) + if window > 0 and penalty > 1.0 and codebooks > 0 + else 0 + for r, window, penalty, codebooks in zip( + batch.reqs, + repetition_windows, + repetition_penalties, + repetition_codebooks, + ) + ] + max_window = max(active_windows, default=0) + if max_window > 0: + repetition_token_ids = torch.full( + (len(batch.reqs), self.n_codebooks, max_window), + -1, + dtype=torch.int64, + device=self.device, + ) + for i, (req, window) in enumerate(zip(batch.reqs, active_windows)): + if window <= 0: + continue + + start = req.device_len - window + if token_pool is None: + history = req.input_ids[start : req.device_len, : self.n_codebooks].to( + device=self.device, dtype=torch.int64, non_blocking=True + ) + else: + history = token_pool[ + req.table_idx, start : req.device_len, : self.n_codebooks + ].to(dtype=torch.int64) + + history = history.transpose(0, 1).contiguous() + valid = (history >= 0) & (history < self.codebook_size) + valid[repetition_codebooks[i] :] = False + repetition_token_ids[i, :, -window:] = torch.where( + valid, history, torch.full_like(history, -1) + ) + + repetition_penalties_tensor = make_device_tensor( + repetition_penalties, torch.float32, self.device + ) + + return TTSBatchSamplingArgs( + temperatures=make_device_tensor(temps, torch.float32, self.device), + top_ks=make_device_tensor(top_ks, torch.int32, self.device), + top_ps=make_device_tensor(top_ps, torch.float32, self.device), + min_ps=make_device_tensor(min_ps, torch.float32, self.device), + text_vocab=self.text_vocab, + repetition_token_ids=repetition_token_ids, + repetition_penalties=repetition_penalties_tensor, + generators=generators, + ) + + def sample( + self, logits: torch.Tensor, args: TTSBatchSamplingArgs + ) -> List[List[int]]: + """Sample from multi-codebook logits. + + Args: + logits: Shape (B, n_codebooks, vocab_size) + args: Sampling arguments + + Returns: + List of unpacked tokens [cb0, cb1, ..., cb8, text_placeholder] + """ + return sample_tts( + logits=logits, + temperatures=args.temperatures, + top_ks=args.top_ks, + top_ps=args.top_ps, + min_ps=args.min_ps, + repetition_token_ids=args.repetition_token_ids, + repetition_penalties=args.repetition_penalties, + text_vocab=args.text_vocab, + generators=args.generators, + ) + + def sample_to_tensor( + self, logits: torch.Tensor, args: TTSBatchSamplingArgs + ) -> torch.Tensor: + """Sample and return as tensor instead of list. + + Args: + logits: Shape (B, n_codebooks, vocab_size) + args: Sampling arguments + + Returns: + Tensor of shape (B, n_codebooks + 1) with sampled tokens + """ + tokens_list = self.sample(logits, args) + return torch.tensor(tokens_list, dtype=torch.int32, device=self.device) diff --git a/zonos2/env.py b/zonos2/env.py new file mode 100644 index 0000000000000000000000000000000000000000..62b9f4c131bad19aec88cd9c1dea9ffeb0533706 --- /dev/null +++ b/zonos2/env.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import os +from functools import partial +from typing import Callable, Generic, TypeVar + + +class BaseEnv: + def _init(self, name: str) -> None: + raise NotImplementedError + + +T = TypeVar("T") + + +class EnvVar(BaseEnv, Generic[T]): + def __init__(self, default_value: T, fn: Callable[[str], T]): + self.value = default_value + self.fn = fn + super().__init__() + + def _init(self, name: str) -> None: + env_value = os.getenv(name) + if env_value is not None: + try: + self.value = self.fn(env_value) + except Exception: + pass + + def __bool__(self): + return self.value + + def __str__(self): + return str(self.value) + + +_TO_BOOL = lambda x: x.lower() in ("1", "true", "yes") + + +def _PARSE_MEM_BYTES(mem: str) -> int: + mem = mem.strip().upper() + if not mem[-1].isalpha(): + return int(mem) + if mem.endswith("B"): + mem = mem[:-1] + UNIT_MAP = {"K": 1024, "M": 1024**2, "G": 1024**3} + return int(float(mem[:-1]) * UNIT_MAP[mem[-1]]) + + +ZONOS2_ENV_PREFIX = "ZONOS2_" +EnvInt = partial(EnvVar[int], fn=int) +EnvFloat = partial(EnvVar[float], fn=float) +EnvBool = partial(EnvVar[bool], fn=_TO_BOOL) +EnvOption = partial(EnvVar[bool | None], fn=_TO_BOOL, default_value=None) +EnvMem = partial(EnvVar[int], fn=_PARSE_MEM_BYTES) + + +class EnvClassSingleton: + _instance: EnvClassSingleton | None = None + + # shell + SHELL_MAX_TOKENS = EnvInt(2048) + SHELL_TOP_K = EnvInt(-1) + SHELL_TOP_P = EnvFloat(1.0) + SHELL_TEMPERATURE = EnvFloat(0.6) + + # backend runtime + FLASHINFER_USE_TENSOR_CORES = EnvOption() + DISABLE_OVERLAP_SCHEDULING = EnvBool(False) + PYNCCL_MAX_BUFFER_SIZE = EnvMem(1024**3) + + def __new__(cls): + # single instance + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self) -> None: + for attr_name in dir(self): + if attr_name.startswith("_"): + continue + attr_value = getattr(self, attr_name) + assert isinstance(attr_value, BaseEnv) + attr_value._init(f"{ZONOS2_ENV_PREFIX}{attr_name}") + + +ENV = EnvClassSingleton() diff --git a/zonos2/kernel/__init__.py b/zonos2/kernel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9eceb180feb31a07adc267aaecb1f846cb6db17b --- /dev/null +++ b/zonos2/kernel/__init__.py @@ -0,0 +1,14 @@ +from .index import indexing +from .pynccl import PyNCCLCommunicator, init_pynccl +from .radix import fast_compare_key +from .store import store_cache +from .tensor import test_tensor + +__all__ = [ + "indexing", + "fast_compare_key", + "store_cache", + "test_tensor", + "init_pynccl", + "PyNCCLCommunicator", +] diff --git a/zonos2/kernel/__main__.py b/zonos2/kernel/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..f28200ef5eeec9e1423507b849fb4bc1d334542b --- /dev/null +++ b/zonos2/kernel/__main__.py @@ -0,0 +1,47 @@ +assert __name__ == "__main__" + + +def generate_clangd(): + import os + import subprocess + + from tvm_ffi.libinfo import find_dlpack_include_path, find_include_path + from zonos2.kernel.utils import DEFAULT_INCLUDE + from zonos2.utils import init_logger + + logger = init_logger(__name__) + logger.info("Generating .clangd file...") + include_paths = [find_include_path(), find_dlpack_include_path()] + DEFAULT_INCLUDE + status = subprocess.run( + args=["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"], + capture_output=True, + check=True, + ) + compute_cap = status.stdout.decode("utf-8").strip().split("\n")[0] + major, minor = compute_cap.split(".") + compile_flags = ",\n ".join( + [ + "-xcuda", + f"--cuda-gpu-arch=sm_{major}{minor}", + "-std=c++20", + "-Wall", + "-Wextra", + ] + + [f"-isystem{path}" for path in include_paths] + ) + clangd_content = f""" +CompileFlags: + Add: [ + {compile_flags} + ] +""" + if os.path.exists(".clangd"): + logger.warning(".clangd file already exists, nothing done.") + logger.warning(f"suggested content: {clangd_content}") + else: + with open(".clangd", "w") as f: + f.write(clangd_content) + logger.info(".clangd file generated.") + + +generate_clangd() diff --git a/zonos2/kernel/csrc/include/zonos2/nccl227.h b/zonos2/kernel/csrc/include/zonos2/nccl227.h new file mode 100644 index 0000000000000000000000000000000000000000..95e474e484c9061534252da42bac111a7ea98884 --- /dev/null +++ b/zonos2/kernel/csrc/include/zonos2/nccl227.h @@ -0,0 +1,571 @@ +/************************************************************************* + * Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef NCCL_H_ +#define NCCL_H_ + +#include +#include +#if CUDART_VERSION >= 11000 +#include +#endif +#if __cplusplus && CUDART_VERSION >= 11080 +#include +#endif + +#define NCCL_MAJOR 2 +#define NCCL_MINOR 27 +#define NCCL_PATCH 1 +#define NCCL_SUFFIX "" + +#define NCCL_VERSION_CODE 22701 +#define NCCL_VERSION(X,Y,Z) (((X) <= 2 && (Y) <= 8) ? (X) * 1000 + (Y) * 100 + (Z) : (X) * 10000 + (Y) * 100 + (Z)) + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Opaque handle to communicator */ +typedef struct ncclComm* ncclComm_t; +typedef struct ncclWindow_vidmem* ncclWindow_t; +#define NCCL_COMM_NULL NULL + +#define NCCL_UNIQUE_ID_BYTES 128 +typedef struct { char internal[NCCL_UNIQUE_ID_BYTES]; } ncclUniqueId; + +/* Error type */ +typedef enum { ncclSuccess = 0, + ncclUnhandledCudaError = 1, + ncclSystemError = 2, + ncclInternalError = 3, + ncclInvalidArgument = 4, + ncclInvalidUsage = 5, + ncclRemoteError = 6, + ncclInProgress = 7, + ncclNumResults = 8 } ncclResult_t; + +#define NCCL_CONFIG_UNDEF_INT INT_MIN +#define NCCL_CONFIG_UNDEF_PTR NULL +#define NCCL_SPLIT_NOCOLOR -1 +#define NCCL_UNDEF_FLOAT -1.0f + +/* Window Registration flags */ +#define NCCL_WIN_DEFAULT 0x00 +#define NCCL_WIN_COLL_SYMMETRIC 0x01 + +#define NCCL_WIN_REQUIRED_ALIGNMENT 4096 + +/* NCCL performance policy */ +#define NCCL_CTA_POLICY_DEFAULT 0x00 +#define NCCL_CTA_POLICY_EFFICIENCY 0x01 +#define NCCL_CTA_POLICY_ZERO 0x02 + +/* ncclCommShrink flags*/ +#define NCCL_SHRINK_DEFAULT 0x00 /* shrink the parent communicator */ +#define NCCL_SHRINK_ABORT 0x01 /* First, terminate ongoing parent operations, and then shrink the parent communicator */ + +/* Communicator configuration. Users can assign value to attributes to specify the + * behavior of a communicator. */ +typedef struct ncclConfig_v22800 { + /* attributes that users should never touch. */ + size_t size; + unsigned int magic; + unsigned int version; + /* attributes that users are able to customize. */ + int blocking; + int cgaClusterSize; + int minCTAs; + int maxCTAs; + const char *netName; + int splitShare; + int trafficClass; + const char *commName; + int collnetEnable; + int CTAPolicy; + int shrinkShare; + int nvlsCTAs; + int nChannelsPerNetPeer; + int nvlinkCentricSched; +} ncclConfig_t; + +/* Config initializer must be assigned to initialize config structure when it is created. + * Not initialized config will result in NCCL error. */ +#define NCCL_CONFIG_INITIALIZER { \ + sizeof(ncclConfig_t), /* size */ \ + 0xcafebeef, /* magic */ \ + NCCL_VERSION(NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH), /* version */ \ + NCCL_CONFIG_UNDEF_INT, /* blocking */ \ + NCCL_CONFIG_UNDEF_INT, /* cgaClusterSize */ \ + NCCL_CONFIG_UNDEF_INT, /* minCTAs */ \ + NCCL_CONFIG_UNDEF_INT, /* maxCTAs */ \ + NCCL_CONFIG_UNDEF_PTR, /* netName */ \ + NCCL_CONFIG_UNDEF_INT, /* splitShare */ \ + NCCL_CONFIG_UNDEF_INT, /* trafficClass */ \ + NCCL_CONFIG_UNDEF_PTR, /* commName */ \ + NCCL_CONFIG_UNDEF_INT, /* collnetEnable */ \ + NCCL_CONFIG_UNDEF_INT, /* CTAPolicy */ \ + NCCL_CONFIG_UNDEF_INT, /* shrinkShare */ \ + NCCL_CONFIG_UNDEF_INT, /* nvlsCTAs */ \ + NCCL_CONFIG_UNDEF_INT, /* nChannelsPerNetPeer */ \ + NCCL_CONFIG_UNDEF_INT, /* nvlinkCentricSched */ \ +} + +/* This struct will be used by ncclGroupSimulateEnd() API to query information about simulation. */ +typedef struct ncclSimInfo_v22200 { + size_t size; + unsigned int magic; + unsigned int version; + float estimatedTime; +} ncclSimInfo_t; + +/* NCCL_SIM_INFO_INITIALIZER must be assigned to initialize simInfo structure when it is created. + * Not initialized simInfo will result in NCCL error. */ +#define NCCL_SIM_INFO_INITIALIZER { \ + sizeof(ncclSimInfo_t), /* size */ \ + 0x74685283, /* magic */ \ + NCCL_VERSION(NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH), /* version */ \ + NCCL_UNDEF_FLOAT /* estimated time */ \ +} + +/* NCCL malloc and free function for all types of NCCL optimizations + * (e.g. user buffer registration). The actual allocated size might + * be larger than requested due to granularity requirement. */ +ncclResult_t ncclMemAlloc(void** ptr, size_t size); +ncclResult_t pncclMemAlloc(void** ptr, size_t size); + +ncclResult_t ncclMemFree(void *ptr); +ncclResult_t pncclMemFree(void *ptr); + +/* Return the NCCL_VERSION_CODE of the NCCL library in the supplied integer. + * This integer is coded with the MAJOR, MINOR and PATCH level of the + * NCCL library + */ +ncclResult_t ncclGetVersion(int *version); +ncclResult_t pncclGetVersion(int *version); + +/* Generates an Id to be used in ncclCommInitRank. ncclGetUniqueId should be + * called once and the Id should be distributed to all ranks in the + * communicator before calling ncclCommInitRank. */ +ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId); +ncclResult_t pncclGetUniqueId(ncclUniqueId* uniqueId); + +/* Create a new communicator (multi thread/process version) with a configuration + * set by users. */ +ncclResult_t ncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config); +ncclResult_t pncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config); + +/* Creates a new communicator (multi thread/process version). + * rank must be between 0 and nranks-1 and unique within a communicator clique. + * Each rank is associated to a CUDA device, which has to be set before calling + * ncclCommInitRank. + * ncclCommInitRank implicitly syncronizes with other ranks, so it must be + * called by different threads/processes or use ncclGroupStart/ncclGroupEnd. */ +ncclResult_t ncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank); +ncclResult_t pncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank); + +/* Creates a clique of communicators (single process version). + * This is a convenience function to create a single-process communicator clique. + * Returns an array of ndev newly initialized communicators in comm. + * comm should be pre-allocated with size at least ndev*sizeof(ncclComm_t). + * If devlist is NULL, the first ndev CUDA devices are used. + * Order of devlist defines user-order of processors within the communicator. */ +ncclResult_t ncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist); +ncclResult_t pncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist); + +/* Finalize a communicator. ncclCommFinalize flushes all issued communications, + * and marks communicator state as ncclInProgress. The state will change to ncclSuccess + * when the communicator is globally quiescent and related resources are freed; then, + * calling ncclCommDestroy can locally free the rest of the resources (e.g. communicator + * itself) without blocking. */ +ncclResult_t ncclCommFinalize(ncclComm_t comm); +ncclResult_t pncclCommFinalize(ncclComm_t comm); + +/* Frees local resources associated with communicator object. */ +ncclResult_t ncclCommDestroy(ncclComm_t comm); +ncclResult_t pncclCommDestroy(ncclComm_t comm); + +/* Frees resources associated with communicator object and aborts any operations + * that might still be running on the device. */ +ncclResult_t ncclCommAbort(ncclComm_t comm); +ncclResult_t pncclCommAbort(ncclComm_t comm); + +/* Creates one or more communicators from an existing one. + * Ranks with the same color will end up in the same communicator. + * Within the new communicator, key will be used to order ranks. + * NCCL_SPLIT_NOCOLOR as color will indicate the rank will not be part of any group + * and will therefore return a NULL communicator. + * If config is NULL, the new communicator will inherit the original communicator's + * configuration*/ +ncclResult_t ncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t *newcomm, ncclConfig_t* config); +ncclResult_t pncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t *newcomm, ncclConfig_t* config); + +/* Shrink existing communicator. + * Ranks in excludeRanksList will be removed form the existing communicator. + * Within the new communicator, ranks will be re-ordered to fill the gap of removed ones. + * If config is NULL, the new communicator will inherit the original communicator's configuration + * The flag enables NCCL to adapt to various states of the parent communicator, see NCCL_SHRINK flags.*/ +ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags); +ncclResult_t pncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags); + +/* Creates a new communicator (multi thread/process version), similar to ncclCommInitRankConfig. + * Allows to use more than one ncclUniqueId (up to one per rank), indicated by nId, to accelerate the init operation. + * The number of ncclUniqueIds and their order must be the same for every rank. + */ +ncclResult_t ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config); +ncclResult_t pncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config); + +/* Returns a string for each error code. */ +const char* ncclGetErrorString(ncclResult_t result); +const char* pncclGetErrorString(ncclResult_t result); + +/* Returns a human-readable message of the last error that occurred. */ +const char* ncclGetLastError(ncclComm_t comm); +const char* pncclGetLastError(ncclComm_t comm); + +/* Reload environment variables that determine logging. */ +__attribute__ ((deprecated("ncclResetDebugInit is not supported as part of the NCCL API and will be removed in the future"))) +void ncclResetDebugInit(); +__attribute__ ((deprecated("pncclResetDebugInit is not supported as part of the NCCL API and will be removed in the future"))) +void pncclResetDebugInit(); + +/* Checks whether the comm has encountered any asynchronous errors */ +ncclResult_t ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t *asyncError); +ncclResult_t pncclCommGetAsyncError(ncclComm_t comm, ncclResult_t *asyncError); + +/* Gets the number of ranks in the communicator clique. */ +ncclResult_t ncclCommCount(const ncclComm_t comm, int* count); +ncclResult_t pncclCommCount(const ncclComm_t comm, int* count); + +/* Returns the cuda device number associated with the communicator. */ +ncclResult_t ncclCommCuDevice(const ncclComm_t comm, int* device); +ncclResult_t pncclCommCuDevice(const ncclComm_t comm, int* device); + +/* Returns the user-ordered "rank" associated with the communicator. */ +ncclResult_t ncclCommUserRank(const ncclComm_t comm, int* rank); +ncclResult_t pncclCommUserRank(const ncclComm_t comm, int* rank); + +/* Register CUDA buffer for zero-copy operation */ +ncclResult_t ncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle); +ncclResult_t pncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle); + +/* Deregister CUDA buffer */ +ncclResult_t ncclCommDeregister(const ncclComm_t comm, void* handle); +ncclResult_t pncclCommDeregister(const ncclComm_t comm, void* handle); + +/* Register memory window */ +ncclResult_t ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags); +ncclResult_t pncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags); + +/* Deregister symmetric memory */ +ncclResult_t ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win); +ncclResult_t pncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win); + +/* Reduction operation selector */ +typedef enum { ncclNumOps_dummy = 5 } ncclRedOp_dummy_t; +typedef enum { ncclSum = 0, + ncclProd = 1, + ncclMax = 2, + ncclMin = 3, + ncclAvg = 4, + /* ncclNumOps: The number of built-in ncclRedOp_t values. Also + * serves as the least possible value for dynamic ncclRedOp_t's + * as constructed by ncclRedOpCreate*** functions. */ + ncclNumOps = 5, + /* ncclMaxRedOp: The largest valid value for ncclRedOp_t. + * It is defined to be the largest signed value (since compilers + * are permitted to use signed enums) that won't grow + * sizeof(ncclRedOp_t) when compared to previous NCCL versions to + * maintain ABI compatibility. */ + ncclMaxRedOp = 0x7fffffff>>(32-8*sizeof(ncclRedOp_dummy_t)) + } ncclRedOp_t; + +/* Data types */ +typedef enum { ncclInt8 = 0, ncclChar = 0, + ncclUint8 = 1, + ncclInt32 = 2, ncclInt = 2, + ncclUint32 = 3, + ncclInt64 = 4, + ncclUint64 = 5, + ncclFloat16 = 6, ncclHalf = 6, + ncclFloat32 = 7, ncclFloat = 7, + ncclFloat64 = 8, ncclDouble = 8, + ncclBfloat16 = 9, + ncclFloat8e4m3 = 10, + ncclFloat8e5m2 = 11, + ncclNumTypes = 12 +} ncclDataType_t; + +/* ncclScalarResidence_t: Location and dereferencing logic for scalar arguments. */ +typedef enum { + /* ncclScalarDevice: The scalar is in device-visible memory and will be + * dereferenced while the collective is running. */ + ncclScalarDevice = 0, + + /* ncclScalarHostImmediate: The scalar is in host-visible memory and will be + * dereferenced before the ncclRedOpCreate***() function returns. */ + ncclScalarHostImmediate = 1 +} ncclScalarResidence_t; + +/* + * ncclRedOpCreatePreMulSum + * + * Creates a new reduction operator which pre-multiplies input values by a given + * scalar locally before reducing them with peer values via summation. For use + * only with collectives launched against *comm* and *datatype*. The + * *residence* argument indicates how/when the memory pointed to by *scalar* + * will be dereferenced. Upon return, the newly created operator's handle + * is stored in *op*. + */ +ncclResult_t ncclRedOpCreatePreMulSum(ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm); +ncclResult_t pncclRedOpCreatePreMulSum(ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm); + +/* + * ncclRedOpDestroy + * + * Destroys the reduction operator *op*. The operator must have been created by + * ncclRedOpCreatePreMul with the matching communicator *comm*. An operator may be + * destroyed as soon as the last NCCL function which is given that operator returns. + */ +ncclResult_t ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm); +ncclResult_t pncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm); + +/* + * Collective communication operations + * + * Collective communication operations must be called separately for each + * communicator in a communicator clique. + * + * They return when operations have been enqueued on the CUDA stream. + * + * Since they may perform inter-CPU synchronization, each call has to be done + * from a different thread or process, or need to use Group Semantics (see + * below). + */ + +/* + * Reduce + * + * Reduces data arrays of length count in sendbuff into recvbuff using op + * operation. + * recvbuff may be NULL on all calls except for root device. + * root is the rank (not the CUDA device) where data will reside after the + * operation is complete. + * + * In-place operation will happen if sendbuff == recvbuff. + */ +ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, + ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, + ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + +/* + * (deprecated) Broadcast (in-place) + * + * Copies count values from root to all other devices. + * root is the rank (not the CUDA device) where data resides before the + * operation is started. + * + * This operation is implicitely in place. + */ +ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, + ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, + ncclComm_t comm, cudaStream_t stream); + +/* + * Broadcast + * + * Copies count values from root to all other devices. + * root is the rank (not the CUDA device) where data resides before the + * operation is started. + * + * In-place operation will happen if sendbuff == recvbuff. + */ +ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, + ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, + ncclComm_t comm, cudaStream_t stream); + +/* + * All-Reduce + * + * Reduces data arrays of length count in sendbuff using op operation, and + * leaves identical copies of result on each recvbuff. + * + * In-place operation will happen if sendbuff == recvbuff. + */ +ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream); + +/* + * Reduce-Scatter + * + * Reduces data in sendbuff using op operation and leaves reduced result + * scattered over the devices so that recvbuff on rank i will contain the i-th + * block of the result. + * Assumes sendcount is equal to nranks*recvcount, which means that sendbuff + * should have a size of at least nranks*recvcount elements. + * + * In-place operations will happen if recvbuff == sendbuff + rank * recvcount. + */ +ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, + size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + cudaStream_t stream); +ncclResult_t pncclReduceScatter(const void* sendbuff, void* recvbuff, + size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + cudaStream_t stream); + +/* + * All-Gather + * + * Each device gathers sendcount values from other GPUs into recvbuff, + * receiving data from rank i at offset i*sendcount. + * Assumes recvcount is equal to nranks*sendcount, which means that recvbuff + * should have a size of at least nranks*sendcount elements. + * + * In-place operations will happen if sendbuff == recvbuff + rank * sendcount. + */ +ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, + ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, + ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); + +/* + * All-to-All + * + * Each device sends count values to all other devices and receives count values + * from all other devices. Data to send to destination rank j is taken from + * sendbuff+j*count and data received from source rank i is placed at + * recvbuff+i*count. + */ +ncclResult_t ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); + +/* + * Gather + * + * Each rank sends count elements from sendbuff to the root rank. + * On the root rank, data from rank i is placed at recvbuff + i*count. + * On non-root ranks, recvbuff is not used. + * root is the rank where data will be gathered. + * + * In-place operations will happen if sendbuff == recvbuff + root * count. + */ +ncclResult_t ncclGather(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclGather(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); + +/* + * Scatter + * + * On the root rank, count elements from sendbuff+i*count are sent to rank i. + * On non-root ranks, sendbuff is not used. + * Each rank receives count elements into recvbuff. + * root is the rank that will distribute the data. + * + * In-place operations will happen if recvbuff == sendbuff + root * count. + */ +ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclScatter(const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); + +/* + * Send + * + * Send data from sendbuff to rank peer. + * + * Rank peer needs to call ncclRecv with the same datatype and the same count from this + * rank. + * + * This operation is blocking for the GPU. If multiple ncclSend and ncclRecv operations + * need to progress concurrently to complete, they must be fused within a ncclGroupStart/ + * ncclGroupEnd section. + */ +ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, + ncclComm_t comm, cudaStream_t stream); +ncclResult_t pncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, + ncclComm_t comm, cudaStream_t stream); + +/* + * Receive + * + * Receive data from rank peer into recvbuff. + * + * Rank peer needs to call ncclSend with the same datatype and the same count to this + * rank. + * + * This operation is blocking for the GPU. If multiple ncclSend and ncclRecv operations + * need to progress concurrently to complete, they must be fused within a ncclGroupStart/ + * ncclGroupEnd section. + */ +ncclResult_t pncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, + ncclComm_t comm, cudaStream_t stream); +ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, + ncclComm_t comm, cudaStream_t stream); + +/* + * Group semantics + * + * When managing multiple GPUs from a single thread, and since NCCL collective + * calls may perform inter-CPU synchronization, we need to "group" calls for + * different ranks/devices into a single call. + * + * Grouping NCCL calls as being part of the same collective operation is done + * using ncclGroupStart and ncclGroupEnd. ncclGroupStart will enqueue all + * collective calls until the ncclGroupEnd call, which will wait for all calls + * to be complete. Note that for collective communication, ncclGroupEnd only + * guarantees that the operations are enqueued on the streams, not that + * the operation is effectively done. + * + * Both collective communication and ncclCommInitRank can be used in conjunction + * of ncclGroupStart/ncclGroupEnd, but not together. + * + * Group semantics also allow to fuse multiple operations on the same device + * to improve performance (for aggregated collective calls), or to permit + * concurrent progress of multiple send/receive operations. + */ + +/* + * Group Start + * + * Start a group call. All calls to NCCL until ncclGroupEnd will be fused into + * a single NCCL operation. Nothing will be started on the CUDA stream until + * ncclGroupEnd. + */ +ncclResult_t ncclGroupStart(); +ncclResult_t pncclGroupStart(); + +/* + * Group End + * + * End a group call. Start a fused NCCL operation consisting of all calls since + * ncclGroupStart. Operations on the CUDA stream depending on the NCCL operations + * need to be called after ncclGroupEnd. + */ +ncclResult_t ncclGroupEnd(); +ncclResult_t pncclGroupEnd(); + +/* + * Group Simulate End + * + * Simulate a ncclGroupEnd() call and return NCCL's simulation info in a struct. + */ +ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo); +ncclResult_t pncclGroupSimulateEnd(ncclSimInfo_t* simInfo); + +#ifdef __cplusplus +} // end extern "C" +#endif + +#endif // end include guard diff --git a/zonos2/kernel/csrc/include/zonos2/tensor.h b/zonos2/kernel/csrc/include/zonos2/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..a8c156e493974828a396d8c54f2466e79c16217c --- /dev/null +++ b/zonos2/kernel/csrc/include/zonos2/tensor.h @@ -0,0 +1,496 @@ +#pragma once +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace host { + +namespace stdr = std::ranges; +namespace stdv = std::views; + +namespace details { + +struct SizeRef; +struct DTypeRef; +struct DeviceRef; + +inline constexpr auto kAnyDeviceID = -1; +inline constexpr auto kAnySize = static_cast(-1); +inline constexpr auto kNullSize = static_cast(0); +inline constexpr auto kNullDType = static_cast(18u); +inline constexpr auto kNullDevice = static_cast(-1); + +template struct dtype_trait {}; + +template struct dtype_trait { + inline static constexpr auto value = + DLDataType{.code = std::is_signed_v ? DLDataTypeCode::kDLInt + : DLDataTypeCode::kDLUInt, + .bits = static_cast(sizeof(T) * 8), + .lanes = 1}; +}; + +template struct dtype_trait { + inline static constexpr auto value = + DLDataType{.code = DLDataTypeCode::kDLFloat, + .bits = static_cast(sizeof(T) * 8), + .lanes = 1}; +}; + +template struct device_trait { + inline static constexpr auto value = + DLDevice{.device_type = Code, .device_id = kAnyDeviceID}; +}; + +template +inline constexpr auto kDTypeList = + std::array{dtype_trait::value...}; + +template +inline constexpr auto kDeviceList = + std::array{device_trait::value...}; + +template struct PrintAbleSpan { + explicit PrintAbleSpan(std::span data) : data(data) {} + std::span data; +}; + +// define DLDataType comparison and printing in root namespace +inline constexpr auto kDeviceStringMap = [] { + constexpr auto map = + std::array, 16>{ + std::pair{DLDeviceType::kDLCPU, "cpu"}, + std::pair{DLDeviceType::kDLCUDA, "cuda"}, + std::pair{DLDeviceType::kDLCUDAHost, "cuda_host"}, + std::pair{DLDeviceType::kDLOpenCL, "opencl"}, + std::pair{DLDeviceType::kDLVulkan, "vulkan"}, + std::pair{DLDeviceType::kDLMetal, "metal"}, + std::pair{DLDeviceType::kDLVPI, "vpi"}, + std::pair{DLDeviceType::kDLROCM, "rocm"}, + std::pair{DLDeviceType::kDLROCMHost, "rocm_host"}, + std::pair{DLDeviceType::kDLExtDev, "ext_dev"}, + std::pair{DLDeviceType::kDLCUDAManaged, "cuda_managed"}, + std::pair{DLDeviceType::kDLOneAPI, "oneapi"}, + std::pair{DLDeviceType::kDLWebGPU, "webgpu"}, + std::pair{DLDeviceType::kDLHexagon, "hexagon"}, + std::pair{DLDeviceType::kDLMAIA, "maia"}, + std::pair{DLDeviceType::kDLTrn, "trn"}, + }; + constexpr auto max_type = stdr::max(map | stdv::keys); + auto result = std::array{}; + for (const auto &[code, name] : map) { + result[static_cast(code)] = name; + } + return result; +}(); + +struct PrintableDevice { + DLDevice device; +}; + +inline auto &operator<<(std::ostream &os, DLDevice device) { + const auto &mapping = kDeviceStringMap; + const auto entry = static_cast(device.device_type); + RuntimeCheck(entry < mapping.size()); + const auto name = mapping[entry]; + RuntimeCheck(!name.empty(), "Unknown device: ", int(device.device_type)); + os << name; + if (device.device_id != kAnyDeviceID) + os << "[" << device.device_id << "]"; + return os; +} + +inline auto &operator<<(std::ostream &os, PrintableDevice pd) { + return os << pd.device; +} + +template +inline auto &operator<<(std::ostream &os, PrintAbleSpan span) { + os << "["; + for (const auto i : stdv::iota(std::size_t{0}, span.data.size())) { + if (i > 0) { + os << ", "; + } + os << span.data[i]; + } + os << "]"; + return os; +} + +} // namespace details + +struct SymbolicSize { +public: + SymbolicSize(std::string_view annotation = {}) + : m_value(details::kNullSize), m_annotation(annotation) {} + + auto get_name() const -> std::string_view { return m_annotation; } + auto set_value(int64_t value) -> void { + RuntimeCheck(!this->has_value(), "Size value already set"); + m_value = value; + } + auto has_value() const -> bool { return m_value != 0; } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> int64_t { + RuntimeCheck(this->has_value(), "Size value is not set"); + return m_value; + } + + SymbolicSize(const SymbolicSize &) = delete; + SymbolicSize &operator=(const SymbolicSize &) = delete; + + auto verify(int64_t size, const char *prefix, int64_t dim) -> void { + if (this->has_value()) { + if (m_value != size) { + [[unlikely]]; + Panic("Size mismatch for ", m_name(prefix, dim), ": expected ", m_value, + " but got ", size); + } + } else { + this->set_value(size); + } + } + + auto value_or_name(const char *prefix, int64_t dim) const -> std::string { + if (const auto value = this->get_value()) { + return std::to_string(*value); + } else { + return m_name(prefix, dim); + } + } + +private: + auto m_name(const char *prefix, int64_t dim) const -> std::string { + const auto annotation = this->get_name(); + std::ostringstream os; + if (annotation.empty()) { + os << prefix << '#' << dim; + } else { + os << annotation << '(' << prefix << '#' << dim << ')'; + } + return std::move(os).str(); + } + + std::int64_t m_value; + std::string_view m_annotation; +}; + +inline auto operator==(DLDevice lhs, DLDevice rhs) -> bool { + return lhs.device_type == rhs.device_type && lhs.device_id == rhs.device_id; +} + +struct SymbolicDType { +public: + SymbolicDType() : m_value({details::kNullDType, 0, 0}) {} + + auto set_value(DLDataType value) -> void { + RuntimeCheck(!this->has_value(), "Dtype value already set"); + RuntimeCheck( + m_check(value), "Dtype value [", value, + "] not in the allowed options: ", details::PrintAbleSpan{m_options}); + m_value = value; + } + auto has_value() const -> bool { return m_value.code != details::kNullDType; } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> DLDataType { + RuntimeCheck(this->has_value(), "Dtype value is not set"); + return m_value; + } + + auto set_options(std::span options) -> void { + m_options = options; + } + template auto set_options() -> void { + m_options = details::kDTypeList; + } + + auto verify(DLDataType dtype) -> void { + if (this->has_value()) { + RuntimeCheck(m_value == dtype, "DType mismatch: expected ", m_value, + " but got ", dtype); + } else { + this->set_value(dtype); + } + } + +private: + auto m_check(DLDataType value) const -> bool { + return stdr::empty(m_options) || + (stdr::find(m_options, value) != stdr::end(m_options)); + } + + std::span m_options; + DLDataType m_value; +}; + +struct SymbolicDevice { +public: + SymbolicDevice() : m_value({details::kNullDevice, details::kAnyDeviceID}) {} + + auto set_value(DLDevice value) -> void { + RuntimeCheck(!this->has_value(), "Device value already set"); + RuntimeCheck( + m_check(value), "Device value [", details::PrintableDevice{value}, + "] not in the allowed options: ", details::PrintAbleSpan{m_options}); + m_value = value; + } + auto has_value() const -> bool { + return m_value.device_type != details::kNullDevice; + } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> DLDevice { + RuntimeCheck(this->has_value(), "Device value is not set"); + return m_value; + } + + auto set_options(std::span options) -> void { + m_options = options; + } + template auto set_options() -> void { + m_options = details::kDeviceList; + } + + auto verify(DLDevice device) -> void { + if (this->has_value()) { + RuntimeCheck(m_value == device, "Device mismatch: expected ", + details::PrintableDevice{m_value}, " but got ", + details::PrintableDevice{device}); + } else { + this->set_value(device); + } + } + +private: + auto m_check(DLDevice value) const -> bool { + return stdr::empty(m_options) || + (stdr::any_of(m_options, [value](const DLDevice &opt) { + // device type must exactly match + if (opt.device_type != value.device_type) + return false; + // device id can be wildcarded + return opt.device_id == details::kAnyDeviceID || + opt.device_id == value.device_id; + })); + } + + std::span m_options; + DLDevice m_value; +}; + +struct TensorMatcher; + +namespace details { + +template struct BaseRef { +public: + BaseRef(const BaseRef &) = delete; + BaseRef &operator=(const BaseRef &) = delete; + + auto operator->() const -> T * { return m_ref; } + auto operator*() const -> T & { return *m_ref; } + auto rebind(T &other) -> void { m_ref = &other; } + + explicit BaseRef() : m_ref(&m_cache), m_cache() {} + BaseRef(T &size) : m_ref(&size), m_cache() {} + +private: + T *m_ref; + T m_cache; +}; + +struct SizeRef : BaseRef { + using BaseRef::BaseRef; + SizeRef(int64_t value) { + if (value != kAnySize) { + (**this).set_value(value); + } else { + // otherwise, we can match any size + } + } +}; + +struct DTypeRef : BaseRef { + using BaseRef::BaseRef; + DTypeRef(DLDataType options) { (**this).set_value(options); } + DTypeRef(std::initializer_list options) { + (**this).set_options(options); + } + DTypeRef(std::span options) { + (**this).set_options(options); + } +}; + +struct DeviceRef : BaseRef { + using BaseRef::BaseRef; + DeviceRef(DLDevice options) { (**this).set_value(options); } + DeviceRef(std::initializer_list options) { + (**this).set_options(options); + } + DeviceRef(std::span options) { + (**this).set_options(options); + } +}; + +} // namespace details + +struct TensorMatcher { +private: + using SizeRef = details::SizeRef; + using DTypeRef = details::DTypeRef; + using DeviceRef = details::DeviceRef; + using Loc_t = std::source_location; + +public: + TensorMatcher(const TensorMatcher &) = delete; + TensorMatcher &operator=(const TensorMatcher &) = delete; + + explicit TensorMatcher(std::initializer_list shape) + : m_shape(shape), m_strides(), m_dtype() {} + + auto with_strides(std::initializer_list strides) && // + -> TensorMatcher && { + // no partial update allowed + RuntimeCheck(m_strides.size() == 0, "Strides already specified"); + RuntimeCheck(m_shape.size() == strides.size(), + "Strides size must match shape size"); + m_strides = strides; + return std::move(*this); + } + + template + auto with_dtype(DTypeRef &&dtype) && -> TensorMatcher && { + m_init_dtype(); + m_dtype.rebind(*dtype); + return std::move(*this); + } + + template + requires(sizeof...(Ts) > 0) + auto with_dtype() && -> TensorMatcher && { + m_init_dtype(); + m_dtype->set_options(); + return std::move(*this); + } + + template + auto with_device(DeviceRef &&device) && -> TensorMatcher && { + m_init_device(); + m_device.rebind(*device); + return std::move(*this); + } + + template + requires(sizeof...(Codes) > 0) + auto with_device() && -> TensorMatcher && { + m_init_device(); + m_device->set_options(); + return std::move(*this); + } + + // once we start verification, we cannot modify anymore + auto verify(tvm::ffi::TensorView view, + Loc_t loc = Loc_t::current()) const && -> const TensorMatcher && { + try { + this->m_verify_impl(view); + } catch (PanicError &e) { + auto oss = std::ostringstream{}; + oss << "Tensor match failed for " << this->debug_str() << " at " + << loc.file_name() << ":" << loc.line() + << "\n- Root cause: " << e.detail(); + throw PanicError(std::move(oss).str()); + } + return std::move(*this); + } + + auto debug_str() const -> std::string { + auto oss = std::ostringstream{}; + oss << "Tensor<"; + std::size_t dim = 0; + for (const auto &size_ref : m_shape) { + if (dim > 0) { + oss << ", "; + } + oss << size_ref->value_or_name("shape", dim++); + } + oss << ">"; + if (m_strides.size() > 0) { + oss << " [strides=<"; + dim = 0; + for (const auto &stride_ref : m_strides) { + if (dim > 0) { + oss << ", "; + } + oss << stride_ref->value_or_name("stride", dim++); + } + oss << ">]"; + } + return std::move(oss).str(); + } + +private: + auto m_verify_impl(tvm::ffi::TensorView view) const -> void { + const auto dim = static_cast(view.dim()); + RuntimeCheck(dim == m_shape.size(), "Tensor dimension mismatch: expected ", + m_shape.size(), " but got ", dim); + for (const auto i : stdv::iota(std::size_t{0}, dim)) { + m_shape[i]->verify(view.size(i), "shape", i); + } + if (this->m_has_strides()) { + for (const auto i : stdv::iota(std::size_t{0}, dim)) { + if (view.size(i) != 1 || !m_strides[i]->has_value()) { + // skip stride check for size 1 dimension + m_strides[i]->verify(view.stride(i), "stride", i); + } + } + } else { + RuntimeCheck(view.is_contiguous(), + "Tensor is not contiguous as expected"); + } + // since we may double verify, we will force to check + m_dtype->verify(view.dtype()); + m_device->verify(view.device()); + } + + auto m_init_dtype() -> void { + RuntimeCheck(!m_has_dtype, "DType already specified"); + m_has_dtype = true; + } + auto m_init_device() -> void { + RuntimeCheck(!m_has_device, "Device already specified"); + m_has_device = true; + } + auto m_has_strides() const -> bool { return !m_strides.empty(); } + + std::span m_shape; + std::span m_strides; + DTypeRef m_dtype; + DeviceRef m_device; + bool m_has_dtype = false; + bool m_has_device = false; +}; + +} // namespace host diff --git a/zonos2/kernel/csrc/include/zonos2/utils.cuh b/zonos2/kernel/csrc/include/zonos2/utils.cuh new file mode 100644 index 0000000000000000000000000000000000000000..75acce14c4af8a8fd1518d175df26567cbef2d5a --- /dev/null +++ b/zonos2/kernel/csrc/include/zonos2/utils.cuh @@ -0,0 +1,144 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include + +namespace device { + +inline constexpr auto kWarpThreads = 32u; + +template +__always_inline __device__ constexpr auto div_ceil(T a, U b) { + return (a + b - 1) / b; +} + +namespace pointer { + +// we only allow void * pointer arithmetic for safety + +template +__always_inline __device__ auto offset(T *ptr, U... offset) -> void * { + static_assert(std::is_same_v, + "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +template +__always_inline __device__ auto offset(const T *ptr, U... offset) -> const + void * { + static_assert(std::is_same_v, + "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +} // namespace pointer + +namespace PDL { + +template __always_inline __device__ void wait() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + } +} + +template __always_inline __device__ void launch() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.launch_dependents;" :::); + } +} + +} // namespace PDL + +} // namespace device + +namespace host { + +inline auto +CUDA_CHECK(::cudaError_t error, + std::source_location location = std::source_location::current()) + -> void { + if (error != ::cudaSuccess) { + [[unlikely]]; + ::host::panic(location, "CUDA error: ", ::cudaGetErrorString(error)); + } +} + +inline auto +CUDA_CHECK(std::source_location location = std::source_location::current()) + -> void { + return CUDA_CHECK(::cudaGetLastError(), location); +} + +template inline void set_smem_once(std::size_t smem_size) { + static const auto last_smem_size = [&] { + CUDA_CHECK(::cudaFuncSetAttribute( + F, ::cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + return smem_size; + }(); + RuntimeCheck( + smem_size <= last_smem_size, + "Dynamic shared memory size exceeds the previously set maximum size: ", + last_smem_size, " bytes"); +} + +struct LaunchKernel { +public: + explicit LaunchKernel(dim3 grid_dim, dim3 block_dim, DLDevice device, + std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_config(s_make_config(grid_dim, block_dim, resolve_device(device), + dynamic_shared_mem_bytes)) {} + + explicit LaunchKernel(dim3 grid_dim, dim3 block_dim, cudaStream_t stream, + std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_config(s_make_config(grid_dim, block_dim, stream, + dynamic_shared_mem_bytes)) {} + + static auto resolve_device(DLDevice device) -> cudaStream_t { + return static_cast( + ::TVMFFIEnvGetStream(device.device_type, device.device_id)); + } + + LaunchKernel(const LaunchKernel &) = delete; + LaunchKernel &operator=(const LaunchKernel &) = delete; + + template + auto operator()(T &&kernel, Args &&...args) const -> void { + CUDA_CHECK( + ::cudaLaunchKernelEx(&m_config, kernel, std::forward(args)...)); + } + + auto with_attr(bool use_pdl) -> LaunchKernel & { + if (use_pdl) { + m_attr_cache.id = ::cudaLaunchAttributeProgrammaticStreamSerialization; + m_attr_cache.val.programmaticStreamSerializationAllowed = 1; + m_config.attrs = &m_attr_cache; + m_config.numAttrs = 1; + } else { + m_config.numAttrs = 0; + } + return *this; + } + +private: + static auto s_make_config(dim3 grid_dim, dim3 block_dim, cudaStream_t stream, + std::size_t smem) -> cudaLaunchConfig_t { + auto config = ::cudaLaunchConfig_t{}; + config.gridDim = grid_dim; + config.blockDim = block_dim; + config.dynamicSmemBytes = smem; + config.stream = stream; + config.numAttrs = 0; + return config; + } + cudaLaunchConfig_t m_config; + cudaLaunchAttribute m_attr_cache; +}; + +} // namespace host diff --git a/zonos2/kernel/csrc/include/zonos2/utils.h b/zonos2/kernel/csrc/include/zonos2/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..001a70fd76d1d9819ac9fb277f7f4638cd74c21f --- /dev/null +++ b/zonos2/kernel/csrc/include/zonos2/utils.h @@ -0,0 +1,122 @@ +#pragma once + +// ref: +// https://forums.developer.nvidia.com/t/c-20s-source-location-compilation-error-when-using-nvcc-12-1/258026/3 +#ifdef __CUDACC__ +#pragma push_macro("__cpp_consteval") +#pragma push_macro("_NODISCARD") +#pragma push_macro("__builtin_LINE") + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbuiltin-macro-redefined" +#define __cpp_consteval 201811L +#pragma clang diagnostic pop + +#ifdef _NODISCARD +#undef _NODISCARD +#define _NODISCARD +#endif + +#define consteval constexpr + +#include + +#undef consteval +#pragma pop_macro("__cpp_consteval") +#pragma pop_macro("_NODISCARD") +#else +#include +#endif + +#include + +#include +#include +#include +#include + +namespace host { + +struct PanicError : public std::runtime_error { +public: + // copy and move constructors + PanicError(std::string msg) : runtime_error(msg), m_message(std::move(msg)) {} + auto detail() const -> std::string_view { + const auto sv = std::string_view{m_message}; + const auto pos = sv.find(": "); + return pos == std::string_view::npos ? sv : sv.substr(pos + 2); + } + +private: + std::string m_message; +}; + +template +[[noreturn]] +inline auto panic(std::source_location location, Args &&...args) -> void { + std::ostringstream os; + os << "Runtime check failed at " << location.file_name() << ":" + << location.line(); + if constexpr (sizeof...(args) > 0) { + os << ": "; + (os << ... << std::forward(args)); + } else { + os << " in " << location.function_name(); + } + throw PanicError(std::move(os).str()); +} + +template struct Panic { + explicit Panic(Args &&...args, std::source_location location = + std::source_location::current()) { + [[unlikely]]; + ::host::panic(location, std::forward(args)...); + } + [[noreturn]] ~Panic() { std::terminate(); } +}; + +template struct RuntimeCheck { + template + explicit RuntimeCheck( + T &&condition, Args &&...args, + std::source_location location = std::source_location::current()) { + if (!condition) { + [[unlikely]]; + ::host::panic(location, std::forward(args)...); + } + } +}; + +template +explicit RuntimeCheck(T &&, Args &&...) -> RuntimeCheck; + +template explicit Panic(Args &&...) -> Panic; + +template +inline constexpr auto div_ceil(T a, U b) { + return (a + b - 1) / b; +} + +inline auto dtype_bytes(DLDataType dtype) -> std::size_t { + return static_cast(dtype.bits / 8); +} + +namespace pointer { + +template +inline auto offset(T *ptr, U... offset) -> void * { + static_assert(std::is_same_v, + "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +template +inline auto offset(const T *ptr, U... offset) -> const void * { + static_assert(std::is_same_v, + "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +} // namespace pointer + +} // namespace host diff --git a/zonos2/kernel/csrc/include/zonos2/warp.cuh b/zonos2/kernel/csrc/include/zonos2/warp.cuh new file mode 100644 index 0000000000000000000000000000000000000000..1a1498ec7def92b60d68075df85099ecd5f00eb2 --- /dev/null +++ b/zonos2/kernel/csrc/include/zonos2/warp.cuh @@ -0,0 +1,80 @@ +#pragma once +#include + +#include + +#include + +namespace device::warp { + +namespace details { + +template inline constexpr auto get_mem_package() { + if constexpr (kUnit == 16) { + return uint4{}; + } else if constexpr (kUnit == 8) { + return uint2{}; + } else if constexpr (kUnit == 4) { + return uint1{}; + } else { + static_assert(kUnit == 16 || kUnit == 8 || kUnit == 4, + "Unsupported memory package size"); + } +} + +inline constexpr auto resolve_unit_size(std::size_t x) -> std::size_t { + if (x % (16 * kWarpThreads) == 0) + return 16; + if (x % (8 * kWarpThreads) == 0) + return 8; + if (x % (4 * kWarpThreads) == 0) + return 4; + return 0; // trigger static assert in _get_mem_package +} + +template +using mem_package_t = decltype(get_mem_package()); + +} // namespace details + +template +__always_inline __device__ void copy(void *__restrict__ dst, + const void *__restrict__ src) { + using Package = details::mem_package_t; + constexpr auto kBytesPerLoop = sizeof(Package) * kWarpThreads; + constexpr auto kLoopCount = kBytes / kBytesPerLoop; + static_assert(kBytes % kBytesPerLoop == 0, + "kBytes must be multiple of 128 bytes"); + + const auto dst_packed = static_cast(dst); + const auto src_packed = static_cast(src); + const auto lane_id = threadIdx.x % kWarpThreads; + +#pragma unroll kLoopCount + for (std::size_t i = 0; i < kLoopCount; ++i) { + const auto j = i * kWarpThreads + lane_id; + dst_packed[j] = src_packed[j]; + } +} + +template +__always_inline __device__ void reset(void *__restrict__ dst) { + using Package = details::mem_package_t; + constexpr auto kBytesPerLoop = sizeof(Package) * kWarpThreads; + constexpr auto kLoopCount = kBytes / kBytesPerLoop; + static_assert(kBytes % kBytesPerLoop == 0, + "warp_copy: kBytes must be multiple of 128 bytes"); + + const auto dst_ = static_cast(dst); + const auto lane_id = threadIdx.x % kWarpThreads; + const auto zero_value = Package{}; + +#pragma unroll kLoopCount + for (std::size_t i = 0; i < kLoopCount; ++i) { + dst_[i * kWarpThreads + lane_id] = zero_value; + } +} + +} // namespace device::warp diff --git a/zonos2/kernel/csrc/jit/index.cu b/zonos2/kernel/csrc/jit/index.cu new file mode 100644 index 0000000000000000000000000000000000000000..f59a497fbcd1eab0e9a09d553b0c09f1c539613f --- /dev/null +++ b/zonos2/kernel/csrc/jit/index.cu @@ -0,0 +1,172 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +struct IndexKernelParams { + void *__restrict__ output; + const void *__restrict__ weight; + const void *__restrict__ indice; + std::size_t num_warps; +}; + +struct MaskedKernelParams { + IndexKernelParams params; + std::size_t start; + std::size_t length; +}; + +template +__global__ __launch_bounds__(kNumThreads, kMaxOccupancy) void // + index_kernel(const __grid_constant__ IndexKernelParams params) { + using namespace device; + constexpr auto kSize = kElementSize; + constexpr auto kSizePerWarp = kSize / kNumSplits; + constexpr auto kWarpPerBlock = static_cast(kNumThreads / 32); + + static_assert(kNumThreads % 32 == 0); + static_assert(std::has_single_bit(kNumSplits)); + static_assert(kElementSize % kNumSplits == 0); + + const auto &[output, weight, indices_, num_warps] = params; + const auto indices = static_cast(indices_); + const auto warp_id = + (threadIdx.x / kWarpThreads) + blockIdx.x * kWarpPerBlock; + PDL::wait(); + + if (warp_id < num_warps) { + const auto pos = indices[warp_id / kNumSplits]; + const auto dst = pointer::offset(output, warp_id * kSizePerWarp); + const auto src = pointer::offset(weight, pos * kSize, + (warp_id % kNumSplits) * kSizePerWarp); + warp::copy(dst, src); + } + + PDL::launch(); +} + +template +__global__ __launch_bounds__(kNumThreads, kMaxOccupancy) void // + masked_index_kernel( + const __grid_constant__ MaskedKernelParams mask_params) { + using namespace device; + constexpr auto kSize = kElementSize; + constexpr auto kSizePerWarp = kSize / kNumSplits; + constexpr auto kWarpPerBlock = static_cast(kNumThreads / 32); + + static_assert(kNumThreads % 32 == 0); + static_assert(std::has_single_bit(kNumSplits)); + static_assert(kElementSize % kNumSplits == 0); + + const auto &[params, start, length] = mask_params; + const auto &[output, weight, indices_, num_warps] = params; + const auto indices = static_cast(indices_); + const auto warp_id = + (threadIdx.x / kWarpThreads) + blockIdx.x * kWarpPerBlock; + + PDL::wait(); + + if (warp_id < num_warps) { + const auto pos = indices[warp_id / kNumSplits] - start; + const auto dst = pointer::offset(output, warp_id * kSizePerWarp); + if (pos < length) { + const auto src = pointer::offset(weight, pos * kSize, + (warp_id % kNumSplits) * kSizePerWarp); + warp::copy(dst, src); + } else { + warp::reset(dst); + } + } + + PDL::launch(); +} + +template +struct IndexKernel { + static void run(const tvm::ffi::TensorView weights, + const tvm::ffi::TensorView indices, + const tvm::ffi::TensorView output, + tvm::ffi::Optional> mask_opts) { + using namespace host; + auto D = SymbolicSize{"D"}; // embedding size + auto L = SymbolicSize{"L"}; // num indices + auto device_ = SymbolicDevice{}; + auto weights_dtype_ = SymbolicDType{}; + auto indices_dtype_ = SymbolicDType{}; + + TensorMatcher({-1, D}) // + .with_dtype(weights_dtype_) + .with_device(device_) + .verify(weights); + TensorMatcher({L, D}) // + .with_dtype(weights_dtype_) + .with_device(device_) + .verify(output); + TensorMatcher({L}) // + .with_dtype(indices_dtype_) + .with_device(device_) + .verify(indices); + + const auto device = device_.unwrap(); + const auto use_int32 = indices_dtype_.unwrap().bits == 32; + const auto num_indices = L.unwrap(); + const auto entry_size = dtype_bytes(weights_dtype_.unwrap()) * D.unwrap(); + RuntimeCheck(entry_size == element_size, + "IndexKernel: element_size mismatch."); + + constexpr auto kWarpPerBlock = num_threads / 32; + const auto num_warps = num_splits * num_indices; + const auto num_blocks = div_ceil(num_warps, kWarpPerBlock); + const auto params = IndexKernelParams{ + .output = static_cast(output.data_ptr()), + .weight = static_cast(weights.data_ptr()), + .indice = indices.data_ptr(), + .num_warps = num_warps, + }; + + if (mask_opts.has_value()) { + const auto &obj = mask_opts.value(); + const auto [start, length] = obj; + const auto m_params = MaskedKernelParams{ + .params = params, + .start = static_cast(start), + .length = static_cast(length), + }; + const auto kernel = + use_int32 ? masked_index_kernel + : masked_index_kernel; + LaunchKernel(num_blocks, num_threads, device) + .with_attr(use_pdl)(kernel, m_params); + } else { + const auto kernel = + use_int32 ? index_kernel + : index_kernel; + LaunchKernel(num_blocks, num_threads, device) + .with_attr(use_pdl)(kernel, params); + } + } +}; + +} // namespace diff --git a/zonos2/kernel/csrc/jit/store.cu b/zonos2/kernel/csrc/jit/store.cu new file mode 100644 index 0000000000000000000000000000000000000000..5a356506dc4559f342a48c472a521eef1b924b98 --- /dev/null +++ b/zonos2/kernel/csrc/jit/store.cu @@ -0,0 +1,123 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace { + +struct StoreKernelParams { + void *__restrict__ k_cache; + void *__restrict__ v_cache; + const void *__restrict__ indices; + const void *__restrict__ k; + const void *__restrict__ v; + std::size_t kv_cache_stride; + std::size_t kv_input_stride; + std::size_t length; +}; + +template +__global__ __launch_bounds__(kNumThreads, kMaxOccupancy) void // + store_kv_cache(const __grid_constant__ StoreKernelParams params) { + using namespace device; + + constexpr auto kWarpPerBlock = + static_cast(kNumThreads / kWarpThreads); + static_assert(kNumThreads % kWarpThreads == 0); + + const auto &[k_cache, v_cache, indices, k, v, kv_cache_stride, + kv_input_stride, length] = params; + const auto warp_id = + (threadIdx.x / kWarpThreads) + blockIdx.x * kWarpPerBlock; + PDL::wait(); + + // each warp handles one element + if (warp_id < length) { + const auto pos = static_cast(indices)[warp_id]; + const auto dst_k = pointer::offset(k_cache, pos * kv_cache_stride); + const auto src_k = pointer::offset(k, warp_id * kv_input_stride); + warp::copy(dst_k, src_k); + const auto dst_v = pointer::offset(v_cache, pos * kv_cache_stride); + const auto src_v = pointer::offset(v, warp_id * kv_input_stride); + warp::copy(dst_v, src_v); + } + + PDL::launch(); +} + +template +struct StoreKernel { + static void run(const tvm::ffi::TensorView k_cache, + const tvm::ffi::TensorView v_cache, + const tvm::ffi::TensorView indices, + const tvm::ffi::TensorView k, const tvm::ffi::TensorView v) { + using namespace host; + auto D = SymbolicSize{"D"}; // element size + auto L = SymbolicSize{"L"}; // length + auto X = SymbolicSize{"X"}; // stride kv cache + auto Y = SymbolicSize{"Y"}; // stride kv input + auto indices_dtype_ = SymbolicDType{}; + auto dtype_ = SymbolicDType{}; + auto device_ = SymbolicDevice{}; + + TensorMatcher({-1, D}) // + .with_strides({X, 1}) + .with_device(device_) + .with_dtype(dtype_) + .verify(k_cache) + .verify(v_cache); + TensorMatcher({L, D}) // + .with_strides({Y, 1}) + .with_device(device_) + .with_dtype(dtype_) + .verify(k) + .verify(v); + TensorMatcher({L}) // + .with_device(device_) + .with_dtype(indices_dtype_) + .verify(indices); + + const auto dtype_size = dtype_bytes(dtype_.unwrap()); + RuntimeCheck(element_size == dtype_size * D.unwrap()); + + const auto device = device_.unwrap(); + const auto use_int32 = indices_dtype_.unwrap().bits == 32; + const auto length = static_cast(L.unwrap()); + const auto kv_cache_stride = X.unwrap() * dtype_size; + const auto kv_input_stride = Y.unwrap() * dtype_size; + + const auto params = StoreKernelParams{ + .k_cache = k_cache.data_ptr(), + .v_cache = v_cache.data_ptr(), + .indices = indices.data_ptr(), + .k = k.data_ptr(), + .v = v.data_ptr(), + .kv_cache_stride = kv_cache_stride, + .kv_input_stride = kv_input_stride, + .length = length, + }; + + constexpr auto kWarpPerBlock = num_threads / 32; + static_assert(num_threads % 32 == 0); + const auto num_blocks = div_ceil(length, kWarpPerBlock); + const auto kernel = use_int32 + ? store_kv_cache + : store_kv_cache; + LaunchKernel(num_blocks, num_threads, device) + .with_attr(use_pdl)(kernel, params); + } +}; + +} // namespace diff --git a/zonos2/kernel/csrc/src/pynccl.cu b/zonos2/kernel/csrc/src/pynccl.cu new file mode 100644 index 0000000000000000000000000000000000000000..14152e2022d132e7f6fb3ec23c2b164af6b50f28 --- /dev/null +++ b/zonos2/kernel/csrc/src/pynccl.cu @@ -0,0 +1,188 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +using NCCLIDList = tvm::ffi::Array; + +auto NCCL_CHECK(::ncclResult_t result) -> void { + if (result != ::ncclSuccess) { + host::RuntimeCheck(false, ::ncclGetErrorString(result)); + } +} + +auto get_uid(const NCCLIDList &wrapper) -> ncclUniqueId { + host::RuntimeCheck(wrapper.size() == NCCL_UNIQUE_ID_BYTES, + "Invalid NCCL ID wrapper size"); + ncclUniqueId id; + std::copy(wrapper.begin(), wrapper.end(), id.internal); + return id; +} + +auto create_uid() -> NCCLIDList { + ncclUniqueId id; + NCCL_CHECK(::ncclGetUniqueId(&id)); + return NCCLIDList(id.internal, id.internal + NCCL_UNIQUE_ID_BYTES); +} + +const auto kNCCLReduceOPMap = std::unordered_map{ + {"sum", ::ncclSum}, {"prod", ::ncclProd}, {"max", ::ncclMax}, + {"min", ::ncclMin}, {"avg", ::ncclAvg}, +}; + +struct DLDataTypeHash { + auto operator()(const DLDataType &dtype) const noexcept -> std::size_t { + return std::bit_cast(dtype); + } +}; + +template +auto operator==(const DLDataType &a, const DLDataType &b) -> bool { + return a.code == b.code && a.bits == b.bits && a.lanes == b.lanes; +} + +const auto kNCCLDtypeMap = + std::unordered_map{ + {{DLDataTypeCode::kDLFloat, 16, 1}, ncclFloat16}, + {{DLDataTypeCode::kDLBfloat, 16, 1}, ncclBfloat16}, + }; + +using std::shared_ptr; + +template using shared_obj = shared_ptr>; +template +inline constexpr auto template_fn = + [](auto &&...args) { return Fn(std::forward(args)...); }; + +struct NCCLWrapper : public tvm::ffi::Object { +public: + NCCLWrapper(int rank, int world_size, const size_t max_bytes, NCCLIDList uid) + : m_rank(rank), m_world_size(world_size), m_max_bytes(max_bytes) { + ncclUniqueId id = get_uid(uid); + ncclComm_t comm; + NCCL_CHECK(::ncclCommInitRank(&comm, m_world_size, id, m_rank)); + m_comm = {comm, template_fn<::ncclCommDestroy>}; + + void *buf; + NCCL_CHECK(::ncclMemAlloc(&buf, max_bytes)); + m_sym_mem = {buf, template_fn<::ncclMemFree>}; + + ncclWindow_t win; + NCCL_CHECK(::ncclCommWindowRegister(comm, buf, max_bytes, &win, + NCCL_WIN_COLL_SYMMETRIC)); + m_win = {win, [comm = m_comm](ncclWindow_t w) { + return NCCL_CHECK(::ncclCommWindowDeregister(comm.get(), w)); + }}; + } + + auto all_reduce(tvm::ffi::TensorView t, std::string op) const -> void { + using namespace host; + RuntimeCheck(t.device().device_type == kDLCUDA, + "Tensor must be on CUDA device"); + RuntimeCheck(t.is_contiguous(), "Tensor must be contiguous"); + const auto size_dim = static_cast(t.shape().Product()); + const auto dtype = kNCCLDtypeMap.at(t.dtype()); + const auto size_bytes = size_dim * (t.dtype().bits / 8); + const auto data_ptr = t.data_ptr(); + const auto reduce_op = kNCCLReduceOPMap.at(op); + const auto stream = LaunchKernel::resolve_device(t.device()); + + if (size_bytes <= m_max_bytes) { // use internal buffer + const auto buf_ptr = m_sym_mem.get(); + const auto need_memcpy = (buf_ptr != data_ptr); + if (need_memcpy) { + CUDA_CHECK(::cudaMemcpyAsync(buf_ptr, data_ptr, size_bytes, + ::cudaMemcpyDeviceToDevice, stream)); + } + NCCL_CHECK(::ncclAllReduce( + /*sendbuff=*/buf_ptr, + /*recvbuff=*/buf_ptr, + /*count=*/size_dim, + /*datatype=*/dtype, + /*op=*/reduce_op, + /*comm=*/m_comm.get(), + /*stream=*/stream)); + if (need_memcpy) { + CUDA_CHECK(::cudaMemcpyAsync(data_ptr, buf_ptr, size_bytes, + ::cudaMemcpyDeviceToDevice, stream)); + } + } else { + NCCL_CHECK(::ncclAllReduce( + /*sendbuff=*/data_ptr, + /*recvbuff=*/data_ptr, + /*count=*/size_dim, + /*datatype=*/dtype, + /*op=*/reduce_op, + /*comm=*/m_comm.get(), + /*stream=*/stream)); + } + } + + auto all_gather(tvm::ffi::TensorView dst, tvm::ffi::TensorView src) const + -> void { + using namespace host; + RuntimeCheck(src.device().device_type == kDLCUDA, + "Tensor must be on CUDA device"); + RuntimeCheck(src.is_contiguous(), "Tensor must be contiguous"); + RuntimeCheck(dst.device().device_type == kDLCUDA, + "Tensor must be on CUDA device"); + RuntimeCheck(dst.is_contiguous(), "Tensor must be contiguous"); + RuntimeCheck(dst.size(0) == src.size(0) * m_world_size, + "Destination tensor has incorrect size"); + const auto size_dim = static_cast(src.shape().Product()); + const auto dtype = kNCCLDtypeMap.at(src.dtype()); + const auto src_ptr = src.data_ptr(); + const auto dst_ptr = dst.data_ptr(); + const auto stream = LaunchKernel::resolve_device(src.device()); + // do not use internal buffer for all_gather, directly gather to output + // tensor + NCCL_CHECK(::ncclAllGather( + /*sendbuff=*/src_ptr, + /*recvbuff=*/dst_ptr, + /*sendcount=*/size_dim, + /*datatype=*/dtype, + /*comm=*/m_comm.get(), + /*stream=*/stream)); + } + + auto get_buffer() const -> void * { return m_sym_mem.get(); } + + TVM_FFI_DECLARE_OBJECT_INFO_FINAL("zonos2.NCCLWrapper", NCCLWrapper, + tvm::ffi::Object); + +private: + int m_rank; + int m_world_size; + size_t m_max_bytes; + shared_obj m_comm; + shared_ptr m_sym_mem; + shared_obj m_win; +}; + +TVM_FFI_STATIC_INIT_BLOCK() { + namespace refl = tvm::ffi::reflection; + refl::ObjectDef() + .def(refl::init(), "__init__") + .def("all_reduce", &NCCLWrapper::all_reduce) + .def("all_gather", &NCCLWrapper::all_gather) + .def("get_buffer", &NCCLWrapper::get_buffer); +} + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(create_nccl_uid, &create_uid); + +} // namespace diff --git a/zonos2/kernel/csrc/src/radix.cpp b/zonos2/kernel/csrc/src/radix.cpp new file mode 100644 index 0000000000000000000000000000000000000000..62e56c8ea9509d67fbd01a714f006238b4af4f8f --- /dev/null +++ b/zonos2/kernel/csrc/src/radix.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +auto _is_1d_cpu_int_tensor(const tvm::ffi::TensorView tensor) -> bool { + return tensor.ndim() == 1 && tensor.is_contiguous() && + tensor.device().device_type == kDLCPU && + (tensor.dtype().code == kDLInt) && + (tensor.dtype().bits == 32 || tensor.dtype().bits == 64); +} + +auto fast_compare_key(const tvm::ffi::TensorView a, + const tvm::ffi::TensorView b) -> size_t { + host::RuntimeCheck(_is_1d_cpu_int_tensor(a) && _is_1d_cpu_int_tensor(b), + "Both tensors must be 1D CPU int tensors."); + host::RuntimeCheck(a.dtype() == b.dtype()); + const auto a_ptr = a.data_ptr(); + const auto b_ptr = b.data_ptr(); + const auto common_len = std::min(a.size(0), b.size(0)); + if (a.dtype().bits == 64) { + const auto a_ptr_64 = static_cast(a_ptr); + const auto b_ptr_64 = static_cast(b_ptr); + const auto diff_pos = + std::mismatch(a_ptr_64, a_ptr_64 + common_len, b_ptr_64); + return static_cast(diff_pos.first - a_ptr_64); + } else { + const auto a_ptr_32 = static_cast(a_ptr); + const auto b_ptr_32 = static_cast(b_ptr); + const auto diff_pos = + std::mismatch(a_ptr_32, a_ptr_32 + common_len, b_ptr_32); + return static_cast(diff_pos.first - a_ptr_32); + } +} + +} // namespace + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(fast_compare_key, fast_compare_key); diff --git a/zonos2/kernel/csrc/src/tensor.cpp b/zonos2/kernel/csrc/src/tensor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c0acd31ffc6c33f43092306b277ac0732ff5b348 --- /dev/null +++ b/zonos2/kernel/csrc/src/tensor.cpp @@ -0,0 +1,31 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +auto test(tvm::ffi::TensorView x, tvm::ffi::TensorView y) -> void { + auto N = host::SymbolicSize{"N"}; + const auto M = 1024; + host::TensorMatcher({N, M}) + .with_strides({-1, 1}) // -1 means any + .with_dtype() + .with_device() + .verify(x); + host::TensorMatcher({N, M}) // default contiguous + .with_dtype({{kDLInt, 32, 1}, {kDLInt, 64, 1}}) + .with_device({{kDLCUDA, 1}}) + .verify(y); + host::RuntimeCheck(N.unwrap() % 4 == 0); +} + +} // namespace + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(test, test); diff --git a/zonos2/kernel/index.py b/zonos2/kernel/index.py new file mode 100644 index 0000000000000000000000000000000000000000..90392f02728f62da5be4447a60bd6ec91334c44f --- /dev/null +++ b/zonos2/kernel/index.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Tuple + +from .utils import KernelConfig, load_jit, make_cpp_args + +if TYPE_CHECKING: + import torch + from tvm_ffi import Module + +DEFAULT_INDEX_KERNEL_CONFIG = KernelConfig(num_threads=128, max_occupancy=1, use_pdl=False) + + +@lru_cache(maxsize=None) +def _jit_index_module( + element_size: int, + *, + num_splits: int = 1, + config: KernelConfig = DEFAULT_INDEX_KERNEL_CONFIG, +) -> Module: + args = make_cpp_args(element_size, num_splits, *config) + return load_jit( + "index", + *args, + cuda_files=["index.cu"], + cuda_wrappers=[("launch", f"IndexKernel<{args}>::run")], + ) + + +def indexing( + weights: torch.Tensor, + indices: torch.Tensor, + *, + output: torch.Tensor | None = None, + vocab_range: Tuple[int, int] | None = None, # (start, length) +) -> torch.Tensor: + if output is None: + output = weights.new_empty(indices.shape[0], weights.shape[1]) + + element_size = weights.shape[1] * weights.element_size() + if element_size % 2048 == 0: + num_splits = 4 + elif element_size % 1024 == 0: + num_splits = 2 + else: + num_splits = 1 + module = _jit_index_module(element_size, num_splits=num_splits) + module.launch(weights, indices, output, vocab_range) + return output diff --git a/zonos2/kernel/moe_impl.py b/zonos2/kernel/moe_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..0e87c7575a50828ffdeb7fd30398440eb7e78739 --- /dev/null +++ b/zonos2/kernel/moe_impl.py @@ -0,0 +1,59 @@ +from typing import Any, Dict + +import torch +import triton +import triton.language as tl +from zonos2.kernel.triton.fused_moe import fused_moe_kernel + + +def fused_moe_kernel_triton( + A: torch.Tensor, + B: torch.Tensor, + C: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + mul_routed_weight: bool, + top_k: int, + config: Dict[str, Any], + compute_type: tl.dtype, +) -> None: + assert topk_weights.stride(1) == 1 + assert sorted_token_ids.stride(0) == 1 + padded_size = 0 + grid = lambda META: ( + triton.cdiv(sorted_token_ids.shape[0], META["BLOCK_SIZE_M"]) + * triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]), + ) + K = B.shape[2] - padded_size + if K % config["BLOCK_SIZE_K"] == 0: + even_Ks = True + else: + even_Ks = False + fused_moe_kernel[grid]( + A, + B, + C, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + B.shape[1], + B.shape[2] - padded_size, + sorted_token_ids.shape[0], + topk_ids.numel(), + A.stride(0), + A.stride(1), + B.stride(0), + B.stride(2), + B.stride(1), + C.stride(1), + C.stride(2), + MUL_ROUTED_WEIGHT=mul_routed_weight, + top_k=top_k, + compute_type=compute_type, + even_Ks=even_Ks, + **config, + ) diff --git a/zonos2/kernel/pynccl.py b/zonos2/kernel/pynccl.py new file mode 100644 index 0000000000000000000000000000000000000000..6ad33368037eabb13c13eafc80d0ab653a7ab3db --- /dev/null +++ b/zonos2/kernel/pynccl.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Literal + +from zonos2.env import ENV + +from .utils import load_aot + +if TYPE_CHECKING: + from abc import abstractmethod + + import torch + from tvm_ffi import Module + + class PyNCCLCommunicator: + @abstractmethod + def all_reduce(self, input: torch.Tensor, op: Literal["sum"]) -> None: ... + @abstractmethod + def all_gather(self, output: torch.Tensor, input: torch.Tensor) -> None: ... + @abstractmethod + def get_buffer(self) -> int: ... + +else: + PyNCCLCommunicator = Any + + +@lru_cache(maxsize=None) +def _load_nccl_module() -> Module: + return load_aot("pynccl", cuda_files=["pynccl.cu"], extra_ldflags=["-lnccl"]) + + +@lru_cache(maxsize=None) +def _get_pynccl_wrapper_cls(): + import tvm_ffi + + @tvm_ffi.register_object("zonos2.NCCLWrapper") + class PyNCCLImpl(tvm_ffi.Object): + def __init__(self, *args): + self.__ffi_init__(*args) + + return PyNCCLImpl + + +def init_pynccl( + *, + tp_rank: int, + tp_size: int, + tp_cpu_group: torch.distributed.ProcessGroup, + max_size_bytes: int = 0, +) -> PyNCCLCommunicator: + import torch + + max_size_bytes = min(max_size_bytes, ENV.PYNCCL_MAX_BUFFER_SIZE.value) + + module = _load_nccl_module() + cls = _get_pynccl_wrapper_cls() + + if tp_rank == 0: + id_list = [module.create_nccl_uid()] + torch.distributed.broadcast_object_list( + id_list, + src=0, + group=tp_cpu_group, + ) + else: + id_list = [None] + torch.distributed.broadcast_object_list( + id_list, + src=0, + group=tp_cpu_group, + ) + + nccl_id = id_list[0] + assert not nccl_id is None, f"Failed to get NCCL unique ID on {tp_rank = }" + + # bypass type checking for the FFI object + return cls(tp_rank, tp_size, max_size_bytes, nccl_id) # type: ignore diff --git a/zonos2/kernel/radix.py b/zonos2/kernel/radix.py new file mode 100644 index 0000000000000000000000000000000000000000..a8b60bab0f405f675ff658070996ab413bdcf824 --- /dev/null +++ b/zonos2/kernel/radix.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING + +from .utils import load_aot + +if TYPE_CHECKING: + import torch + from tvm_ffi import Module + + +@lru_cache(maxsize=None) +def _load_radix_module() -> Module: + return load_aot("radix", cpp_files=["radix.cpp"]) + + +def fast_compare_key(x: torch.Tensor, y: torch.Tensor) -> int: + # compare 2 1-D int cpu tensors for equality + return _load_radix_module().fast_compare_key(x, y) diff --git a/zonos2/kernel/store.py b/zonos2/kernel/store.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd9e41918a57554c43d42e333f68c8b2bdad901 --- /dev/null +++ b/zonos2/kernel/store.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING + +from .utils import KernelConfig, load_jit, make_cpp_args + +if TYPE_CHECKING: + import torch + from tvm_ffi import Module + +DEFAULT_INDEX_KERNEL_CONFIG = KernelConfig(num_threads=128, max_occupancy=1, use_pdl=False) + + +@lru_cache(maxsize=None) +def _jit_store_module( + element_size: int, + *, + config: KernelConfig = DEFAULT_INDEX_KERNEL_CONFIG, +) -> Module: + args = make_cpp_args(element_size, *config) + return load_jit( + "store", + *args, + cuda_files=["store.cu"], + cuda_wrappers=[("launch", f"StoreKernel<{args}>::run")], + ) + + +def store_cache( + k_cache: torch.Tensor, + v_cache: torch.Tensor, + indices: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, +) -> None: + num_tokens = k_cache.shape[0] + k_cache = k_cache.view(num_tokens, -1) + v_cache = v_cache.view(num_tokens, -1) + element_size = k_cache.shape[1] * k_cache.element_size() + module = _jit_store_module(element_size) + module.launch(k_cache, v_cache, indices, k, v) diff --git a/zonos2/kernel/tensor.py b/zonos2/kernel/tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..cbdf987aaf02a2b4f495d8ae7901990e0796ff0c --- /dev/null +++ b/zonos2/kernel/tensor.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING + +from .utils import load_aot + +if TYPE_CHECKING: + import torch + from tvm_ffi import Module + + +@lru_cache(maxsize=None) +def _load_test_tensor_module() -> Module: + return load_aot("test_tensor", cpp_files=["tensor.cpp"]) + + +def test_tensor(x: torch.Tensor, y: torch.Tensor) -> int: + return _load_test_tensor_module().test(x, y) diff --git a/zonos2/kernel/triton/fused_moe.py b/zonos2/kernel/triton/fused_moe.py new file mode 100644 index 0000000000000000000000000000000000000000..3f1ae653174c96f3242e3d6728d971db9247b159 --- /dev/null +++ b/zonos2/kernel/triton/fused_moe.py @@ -0,0 +1,230 @@ +import torch +import triton +import triton.language as tl + + +@triton.jit +def _moe_sum_reduce_kernel( + input_ptr, + input_stride_0, + input_stride_1, + input_stride_2, + output_ptr, + output_stride_0, + output_stride_1, + token_num: int, + topk_num: int, + hidden_dim: int, + routed_scaling_factor: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DIM: tl.constexpr, + NUM_STAGE: tl.constexpr, +): + input_stride_0 = tl.cast(input_stride_0, dtype=tl.int64) + input_stride_1 = tl.cast(input_stride_1, dtype=tl.int64) + output_stride_0 = tl.cast(output_stride_0, dtype=tl.int64) + + token_block_id = tl.program_id(0) + dim_block_id = tl.program_id(1) + + token_start = token_block_id * BLOCK_M + token_end = min((token_block_id + 1) * BLOCK_M, token_num) + + dim_start = dim_block_id * BLOCK_DIM + dim_end = min((dim_block_id + 1) * BLOCK_DIM, hidden_dim) + + offs_dim = dim_start + tl.arange(0, BLOCK_DIM) + + for token_index in range(token_start, token_end): + accumulator = tl.zeros((BLOCK_DIM,), dtype=tl.float32) + input_t_ptr = input_ptr + token_index * input_stride_0 + offs_dim + for i in tl.range(0, topk_num, num_stages=NUM_STAGE): + tmp = tl.load(input_t_ptr + i * input_stride_1, mask=offs_dim < dim_end, other=0.0) + accumulator += tmp + accumulator = accumulator * routed_scaling_factor + store_t_ptr = output_ptr + token_index * output_stride_0 + offs_dim + tl.store( + store_t_ptr, + accumulator.to(input_ptr.dtype.element_ty), + mask=offs_dim < dim_end, + ) + + +@triton.jit +def fused_moe_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + topk_weights_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + num_tokens_post_padded_ptr, + # Matrix dimensions + N, + K, + EM, + num_valid_tokens, + # The stride variables represent how much to increase the ptr by when + # moving by 1 element in a particular dimension. E.g. `stride_am` is + # how much to increase `a_ptr` by to get the element one row down + # (A has M rows). + stride_am, + stride_ak, + stride_be, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + MUL_ROUTED_WEIGHT: tl.constexpr, + top_k: tl.constexpr, + compute_type: tl.constexpr, + even_Ks: tl.constexpr, +): + """ + Implements the fused computation for a Mixture of Experts (MOE) using + token and expert matrices. + + Key Parameters: + - A: The input tensor representing tokens with shape (*, K), where '*' can + be any shape representing batches and K is the feature dimension of + each token. + - B: The stacked MOE weight tensor with shape (E, N, K), where E is + the number of experts, K is the input feature dimension, and N is + the output feature dimension. + - C: The output cache tensor with shape (M, topk, N), where M is the + total number of tokens post padding, topk is the number of times + each token is repeated, and N is the output feature dimension. + - sorted_token_ids: A tensor containing the sorted indices of tokens, + repeated topk times and arranged by the expert index they are + assigned to. + - expert_ids: A tensor containing the indices of the expert for each + block. It determines which expert matrix from B should be used for + each block in A. + + This kernel performs the multiplication of a token by its corresponding + expert matrix as determined by `expert_ids`. The sorting of + `sorted_token_ids` by expert index and padding ensures divisibility by + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix + multiplication across different blocks processed by the same expert. + """ + # ----------------------------------------------------------- + # Map program ids `pid` to the block of C it should compute. + # This is done in a grouped ordering to promote L2 data reuse. + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + # ---------------------------------------------------------- + # Create pointers for the first blocks of A and B. + # We will advance this pointer as we move in the K direction + # and accumulate + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr) + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: + return + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id) + offs_token = offs_token.to(tl.int64) + token_mask = offs_token < num_valid_tokens + + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak) + + off_experts = tl.load(expert_ids_ptr + pid_m) + b_ptrs = ( + b_ptr + + off_experts * stride_be + + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + ) + + # ----------------------------------------------------------- + # Iterate to compute a block of the C matrix. + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block + # of fp32 values for higher accuracy. + # `accumulator` will be converted back to fp16 after the loop. + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + # Load the next block of A and B, generate a mask by checking the + # K dimension. + if even_Ks: + a = tl.load( + a_ptrs, + mask=token_mask[:, None], + other=0.0, + ) + b = tl.load(b_ptrs) + else: + a = tl.load( + a_ptrs, + mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), + other=0.0, + ) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + # We accumulate along the K dimension. + + accumulator += tl.dot(a, b) + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if MUL_ROUTED_WEIGHT: + moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0) + accumulator = accumulator * moe_weight[:, None] + + accumulator = accumulator.to(compute_type) + # ----------------------------------------------------------- + # Write back the block of the output + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +def moe_sum_reduce_triton(input: torch.Tensor, output: torch.Tensor, routed_scaling_factor: float): + assert input.is_contiguous() + assert output.is_contiguous() + + token_num, topk_num, hidden_dim = input.shape + assert output.shape[0] == token_num and output.shape[1] == hidden_dim + + BLOCK_M = 1 + BLOCK_DIM = 2048 + NUM_STAGE = 1 + num_warps = 8 + + grid = ( + triton.cdiv(token_num, BLOCK_M), + triton.cdiv(hidden_dim, BLOCK_DIM), + ) + + _moe_sum_reduce_kernel[grid]( + input, + *input.stride(), + output, + *output.stride(), + token_num=token_num, + topk_num=topk_num, + hidden_dim=hidden_dim, + routed_scaling_factor=routed_scaling_factor, + BLOCK_M=BLOCK_M, + BLOCK_DIM=BLOCK_DIM, + NUM_STAGE=NUM_STAGE, + num_warps=num_warps, + ) + return diff --git a/zonos2/kernel/utils.py b/zonos2/kernel/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..55ae7af357a20a95611f8869b33912494907e908 --- /dev/null +++ b/zonos2/kernel/utils.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import pathlib +from typing import TYPE_CHECKING, List, NamedTuple, Tuple, TypeAlias, Union + +if TYPE_CHECKING: + from tvm_ffi import Module + +KERNEL_PATH = pathlib.Path(__file__).parent / "csrc" +DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")] +DEFAULT_CFLAGS = ["-std=c++20", "-O3"] +DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"] +DEFAULT_LDFLAGS = [] +CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool] + + +class CppArgList(list[str]): + def __str__(self) -> str: + return ", ".join(self) + + +class KernelConfig(NamedTuple): + num_threads: int + max_occupancy: int + use_pdl: bool + + @property + def template_args(self) -> str: + pdl = "true" if self.use_pdl else "false" + return f"{self.num_threads},{self.max_occupancy},{pdl}" + + +def _make_name(*args: str) -> str: + return "zonos2__" + "_".join(str(arg) for arg in args) + + +def _make_wrapper(tup: Tuple[str, str]) -> str: + export_name, kernel_name = tup + return f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({export_name}, ({kernel_name}));" + + +def make_cpp_args(*args: CPP_TEMPLATE_TYPE) -> CppArgList: + def _convert(arg: CPP_TEMPLATE_TYPE) -> str: + if isinstance(arg, bool): + return "true" if arg else "false" + if isinstance(arg, (int, float)): + return str(arg) + raise TypeError(f"Unsupported argument type for cpp template: {type(arg)}") + + return CppArgList(_convert(arg) for arg in args) + + +def load_aot( + *args: str, + cpp_files: List[str] | None = None, + cuda_files: List[str] | None = None, + extra_cflags: List[str] | None = None, + extra_cuda_cflags: List[str] | None = None, + extra_ldflags: List[str] | None = None, + extra_include_paths: List[str] | None = None, + build_directory: str | None = None, +) -> Module: + from tvm_ffi.cpp import load + + cpp_files = cpp_files or [] + cuda_files = cuda_files or [] + extra_cflags = extra_cflags or [] + extra_cuda_cflags = extra_cuda_cflags or [] + extra_ldflags = extra_ldflags or [] + extra_include_paths = extra_include_paths or [] + + cpp_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cpp_files] + cuda_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cuda_files] + + return load( + _make_name(*args), + cpp_files=cpp_files, + cuda_files=cuda_files, + extra_cflags=DEFAULT_CFLAGS + extra_cflags, + extra_cuda_cflags=DEFAULT_CUDA_CFLAGS + extra_cuda_cflags, + extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags, + extra_include_paths=DEFAULT_INCLUDE + extra_include_paths, + build_directory=build_directory, + ) + + +def load_jit( + *args: str, + cpp_files: List[str] | None = None, + cuda_files: List[str] | None = None, + cpp_wrappers: List[Tuple[str, str]] | None = None, + cuda_wrappers: List[Tuple[str, str]] | None = None, + extra_cflags: List[str] | None = None, + extra_cuda_cflags: List[str] | None = None, + extra_ldflags: List[str] | None = None, + extra_include_paths: List[str] | None = None, + build_directory: str | None = None, +) -> Module: + from tvm_ffi.cpp import load_inline + + cpp_files = cpp_files or [] + cuda_files = cuda_files or [] + cpp_wrappers = cpp_wrappers or [] + cuda_wrappers = cuda_wrappers or [] + extra_cflags = extra_cflags or [] + extra_cuda_cflags = extra_cuda_cflags or [] + extra_ldflags = extra_ldflags or [] + extra_include_paths = extra_include_paths or [] + + # include cpp files + cpp_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cpp_files] + cpp_sources = [f'#include "{path}"' for path in cpp_paths] + cpp_sources += [_make_wrapper(tup) for tup in cpp_wrappers] + + # include cuda files + cuda_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cuda_files] + cuda_sources = [f'#include "{path}"' for path in cuda_paths] + cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers] + + return load_inline( + _make_name(*args), + cpp_sources=cpp_sources, + cuda_sources=cuda_sources, + extra_cflags=DEFAULT_CFLAGS + extra_cflags, + extra_cuda_cflags=DEFAULT_CUDA_CFLAGS + extra_cuda_cflags, + extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags, + extra_include_paths=DEFAULT_INCLUDE + extra_include_paths, + build_directory=build_directory, + ) diff --git a/zonos2/kvcache/__init__.py b/zonos2/kvcache/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ad2d8feed84213ed162ef715ac7edf2724e57b67 --- /dev/null +++ b/zonos2/kvcache/__init__.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol + +from zonos2.utils import Registry + +if TYPE_CHECKING: + import torch + from zonos2.models import ModelConfig + +from .base import ( + BaseCacheHandle, + BaseCacheManager, + BaseKVCache, + KVCacheLayout, + SizeInfo, +) + + +class CacheManagerCreator(Protocol): + def __call__(self, device: torch.device) -> BaseCacheManager: ... + + +SUPPORTED_CACHE_MANAGER = Registry[CacheManagerCreator]("Cache Manager") + + +def create_kvcache( + model_config: ModelConfig, + num_pages: int, + dtype: torch.dtype, + device: torch.device, + cache_layout: KVCacheLayout = KVCacheLayout.LayerFirst, +) -> BaseKVCache: + from .mha_pool import MHAKVCache # TODO: support other variants (e.g. MLA) + + return MHAKVCache( + num_kv_heads=model_config.num_kv_heads, + num_pages=num_pages, + kv_layout=cache_layout, + num_layers=model_config.num_layers, + head_dim=model_config.head_dim, + device=device, + dtype=dtype, + ) + + +@SUPPORTED_CACHE_MANAGER.register("naive") +def create_naive_cache_manager(device: torch.device): + from .naive_manager import NaiveCacheManager + + return NaiveCacheManager(device=device) + + +@SUPPORTED_CACHE_MANAGER.register("radix") +def create_radix_cache_manager(device: torch.device): + from .radix_manager import RadixCacheManager + + return RadixCacheManager(device=device) + + +def create_cache_manager(device: torch.device, type: str) -> BaseCacheManager: + return SUPPORTED_CACHE_MANAGER[type](device) + + +__all__ = [ + "create_kvcache", + "create_cache_manager", + "BaseKVCache", + "KVCacheLayout", + "BaseCacheHandle", + "BaseCacheManager", + "SizeInfo", + "SUPPORTED_CACHE_MANAGER", +] diff --git a/zonos2/kvcache/base.py b/zonos2/kvcache/base.py new file mode 100644 index 0000000000000000000000000000000000000000..5e73a7bb1dd49f74d04c1d2c576611ac8323318c --- /dev/null +++ b/zonos2/kvcache/base.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import enum +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import NamedTuple, Tuple + +import torch + + +class BaseKVCache(ABC): + """ + Base class for key-value caches. + This class defines the interface for key-value caches used. + """ + + @abstractmethod + def k_cache(self, index: int) -> torch.Tensor: ... + + @abstractmethod + def v_cache(self, index: int) -> torch.Tensor: ... + + @abstractmethod + def store_kv( + self, k: torch.Tensor, v: torch.Tensor, out_loc: torch.Tensor, layer_id: int + ) -> None: ... + + @property + @abstractmethod + def device(self) -> torch.device: ... + + @property + @abstractmethod + def dtype(self) -> torch.dtype: ... + + @property + @abstractmethod + def num_layers(self) -> int: ... + + +class KVCacheLayout(enum.Enum): + LayerFirst = enum.auto() + PageFirst = enum.auto() + + +@dataclass(frozen=True) +class BaseCacheHandle(ABC): + cached_len: int + + +class SizeInfo(NamedTuple): + evictable_size: int + protected_size: int + + @property + def total_size(self) -> int: + return self.evictable_size + self.protected_size + + +class BaseCacheManager(ABC): + @abstractmethod + def match_prefix(self, input_ids: torch.Tensor) -> Tuple[BaseCacheHandle, torch.Tensor]: + """ + Match prefix and return the indices of the matched prefix in the cache. + This operation will not modify the cache. + The returned indices is only safe to use when the handle is locked. + + Args: + input_ids (torch.Tensor): The input ids to match. Shape: (seq_len,) + Returns: + handle (BaseCacheHandle): The handle to the matched prefix. + indices (torch.Tensor): The indices of the longest-matched prefix in the cache. + """ + + @abstractmethod + def lock_handle(self, handle: BaseCacheHandle, unlock: bool = False) -> None: + """ + Lock or unlock a cache handle. + This operation will not modify the cache, but change the size info only. + When a handle is locked, it cannot be evicted. + Handles must be locked before the previously-returned tensor of `match_prefix` is used. + Otherwise it may be evicted by calling evict. + + Args: + handle (BaseCacheHandle): The cache handle to lock or unlock. + unlock (bool): Whether to unlock the handle. Defaults to False. + """ + + @abstractmethod + def insert_prefix(self, input_ids: torch.Tensor, indices: torch.Tensor) -> int: + """ + Insert a new prefix into the cache. + This operation will modify the cache. + Args: + input_ids (torch.Tensor): The input ids to insert. Shape: (seq_len,) + indices (torch.Tensor): The indices to store the new prefix. Shape: (seq_len,) + + Returns: + int: The length of prefix that is already in the cache. This part is not + inserted, so the caller should free these indices. + """ + + @abstractmethod + def evict(self, size: int) -> torch.Tensor: + """ + Evict some prefixes from the cache to free up space. + This operation will modify the cache. + Note that evict 0 is always safe and does nothing. + Note that the actual evict size may be larger than the requested size. + Args: + size (int): The size to evict. + + Returns: + torch.Tensor: The indices evicted. Shape: (evict_size,) + Raises: + RuntimeError: If the requested size is larger than the evictable size. + """ + + @abstractmethod + def reset(self) -> None: + """Reset the cache manager and the underlying cache.""" + + @property + @abstractmethod + def size_info(self) -> SizeInfo: + """Get the size information of the cache.""" + + @abstractmethod + def check_integrity(self) -> None: + """Check the integrity of the cache. Raise an error if the cache is corrupted.""" diff --git a/zonos2/kvcache/mha_pool.py b/zonos2/kvcache/mha_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..9cbf4e39a4c8ce5509a40355165996779f06e7dc --- /dev/null +++ b/zonos2/kvcache/mha_pool.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import torch +from zonos2.distributed import get_tp_info +from zonos2.utils import divide_even + +from .base import BaseKVCache, KVCacheLayout + + +class MHAKVCache(BaseKVCache): + """ + Base class for key-value caches. + This class defines the interface for key-value caches used in LLMs. + """ + + def __init__( + self, + num_kv_heads: int, + num_layers: int, + head_dim: int, + num_pages: int, + dtype: torch.dtype, + kv_layout: KVCacheLayout, + device: torch.device, + ): + tp_info = get_tp_info() + local_kv_heads = divide_even(num_kv_heads, tp_info.size) + match kv_layout: + case KVCacheLayout.PageFirst: + kv_buffer = torch.empty( + (2, num_pages, num_layers, local_kv_heads, head_dim), + device=device, + dtype=dtype, + ).permute(0, 2, 1, 3, 4) + case KVCacheLayout.LayerFirst: + kv_buffer = torch.empty( + (2, num_layers, num_pages, local_kv_heads, head_dim), + device=device, + dtype=dtype, + ) + case _: + raise ValueError(f"Unsupported kv_layout: {kv_layout}") + self._kv_buffer = kv_buffer.view(2, num_layers, num_pages, 1, local_kv_heads, head_dim) + self._num_layers = num_layers + self._k_buffer = self._kv_buffer[0] + self._v_buffer = self._kv_buffer[1] + self._device = device + self._storage_shape = (num_pages, local_kv_heads, head_dim) + + def k_cache(self, index: int) -> torch.Tensor: + return self._k_buffer[index] + + def v_cache(self, index: int) -> torch.Tensor: + return self._v_buffer[index] + + def store_kv( + self, k: torch.Tensor, v: torch.Tensor, out_loc: torch.Tensor, layer_id: int + ) -> None: + from zonos2.kernel import store_cache + + store_cache( + k_cache=self._k_buffer[layer_id].view(self._storage_shape), + v_cache=self._v_buffer[layer_id].view(self._storage_shape), + indices=out_loc, + k=k, + v=v, + ) + + @property + def device(self) -> torch.device: + return self._device + + @property + def dtype(self) -> torch.dtype: + return self._kv_buffer.dtype + + @property + def num_layers(self) -> int: + return self._num_layers diff --git a/zonos2/kvcache/naive_manager.py b/zonos2/kvcache/naive_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b6d9910700c5f780a87b5caa308e881a18212933 --- /dev/null +++ b/zonos2/kvcache/naive_manager.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import Tuple + +import torch + +from .base import BaseCacheHandle, BaseCacheManager, SizeInfo + + +class NaiveCacheHandle(BaseCacheHandle): + pass + + +class NaiveCacheManager(BaseCacheManager): + def __init__(self, device: torch.device): + self.device = device + self.empty_tensor = torch.empty(0, dtype=torch.int32, device=device) + super().__init__() + + def match_prefix(self, input_ids: torch.Tensor) -> Tuple[NaiveCacheHandle, torch.Tensor]: + _ = input_ids # unused + return NaiveCacheHandle(0), self.empty_tensor + + def lock_handle(self, handle: BaseCacheHandle, unlock: bool = False) -> None: + _ = handle, unlock # unused + + def insert_prefix(self, input_ids: torch.Tensor, indices: torch.Tensor) -> int: + assert len(indices) == len(input_ids) + return len(indices) + + def evict(self, size: int) -> torch.Tensor: + if size == 0: + return self.empty_tensor + raise NotImplementedError("NaiveCacheManager does not support eviction.") + + def reset(self) -> None: + pass + + @property + def size_info(self) -> SizeInfo: + return SizeInfo(evictable_size=0, protected_size=0) + + def check_integrity(self) -> None: + pass diff --git a/zonos2/kvcache/radix_manager.py b/zonos2/kvcache/radix_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..461492b509fb2cad2a250270275213dc47e405af --- /dev/null +++ b/zonos2/kvcache/radix_manager.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import heapq +import time +from dataclasses import dataclass +from typing import Dict, List, Tuple + +import torch + +from .base import BaseCacheHandle, BaseCacheManager, SizeInfo + + +class RadixTreeNode: + counter: int = 0 + + def __init__(self, tic: int | None = None) -> None: + self.children: Dict[int, RadixTreeNode] = {} + self._parent: RadixTreeNode | None = None + self.ref_count: int = 0 + self.uuid = RadixTreeNode.counter + RadixTreeNode.counter += 1 + self.timestamp = tic or time.monotonic_ns() + + # these fields should be updated later + self._key: torch.Tensor + self._value: torch.Tensor + self._length: int + + def set_key_value(self, key: torch.Tensor, value: torch.Tensor) -> None: + assert len(key) == len(value) + self._key = key + self._value = value + self._length = len(key) + + def set_parent(self, parent: RadixTreeNode) -> None: + self._parent = parent + parent.children[int(self._key[0].item())] = self + + @property + def length(self) -> int: + return self._length + + @property + def parent(self) -> RadixTreeNode: + assert self._parent is not None + return self._parent + + @property + def value(self) -> torch.Tensor: + return self._value + + def is_root(self) -> bool: + return self._parent is None + + def is_leaf(self) -> bool: + return len(self.children) == 0 + + def get_match_len(self, input_ids: torch.Tensor) -> int: + from zonos2.kernel import fast_compare_key + + # compare key and input_ids, find the first diff + return fast_compare_key(self._key, input_ids) + + def _split_at(self, pos: int) -> RadixTreeNode: + assert 0 < pos < self.length + parent = self.parent + + new_node = RadixTreeNode(self.timestamp) + new_node.set_key_value(self._key[:pos], self._value[:pos]) + new_node.set_parent(parent) + new_node.ref_count = self.ref_count + + self.set_key_value(self._key[pos:], self._value[pos:]) + self.set_parent(new_node) + + return new_node + + def __lt__(self, other: RadixTreeNode) -> bool: + return self.timestamp < other.timestamp + + +@dataclass(frozen=True) +class RadixCacheHandle(BaseCacheHandle): + node: RadixTreeNode + + +class RadixCacheManager(BaseCacheManager): + def __init__(self, device: torch.device): + self.device = device + self.empty_tensor = torch.empty(0, dtype=torch.int32, device=device) + super().__init__() + self.root_node = RadixTreeNode() + self.root_node.ref_count = 1 # root is always protected + self.evictable_size = 0 + self.protected_size = 0 + + def lock_handle(self, handle: BaseCacheHandle, unlock: bool = False) -> None: + assert isinstance(handle, RadixCacheHandle) + node = handle.node + if unlock: + while not node.is_root(): + node.ref_count -= 1 + assert node.ref_count >= 0 + if node.ref_count == 0: + self.evictable_size += node.length + self.protected_size -= node.length + node = node.parent + else: + while not node.is_root(): + if node.ref_count == 0: + self.evictable_size -= node.length + self.protected_size += node.length + node.ref_count += 1 + node = node.parent + + def match_prefix(self, input_ids: torch.Tensor) -> Tuple[RadixCacheHandle, torch.Tensor]: + node, prefix_len = self._walk(input_ids) + if prefix_len == 0: + assert node.is_root() and node is self.root_node and prefix_len == 0 + return RadixCacheHandle(prefix_len, node), self.empty_tensor + value_list: List[torch.Tensor] = [] + matched_node = node + while not node.is_root(): + value_list.append(node.value) + node = node.parent + value_list.reverse() + return RadixCacheHandle(prefix_len, matched_node), torch.cat(value_list) + + def insert_prefix(self, input_ids: torch.Tensor, indices: torch.Tensor) -> int: + node, prefix_len = self._walk(input_ids) + assert prefix_len <= len(input_ids) + if prefix_len < len(input_ids): + new_node = RadixTreeNode() + new_node.set_key_value(input_ids[prefix_len:], indices[prefix_len:]) + new_node.set_parent(node) + self.evictable_size += new_node.length + return prefix_len + + def _walk(self, input_ids: torch.Tensor) -> Tuple[RadixTreeNode, int]: + prefix_len = 0 + indice_len = len(input_ids) + node = self.root_node + tic = time.monotonic_ns() + + while prefix_len < indice_len: + this_id = int(input_ids[prefix_len].item()) + if this_id not in node.children: + return node, prefix_len + + node = node.children[this_id] + + # NOTE: at least 1 char is matched, so match_len >= 1 + match_len = node.get_match_len(input_ids[prefix_len:]) + prefix_len += match_len + + # need to split the node if not fully matched + if match_len != node.length: + node = node._split_at(match_len) + return node, prefix_len + + # update timestamp for accessed node + node.timestamp = tic + + return node, prefix_len + + def evict(self, size: int) -> torch.Tensor: + if size == 0: + return self.empty_tensor + assert ( + size <= self.evictable_size + ), f"Cannot evict {size}, only {self.evictable_size} is evictable" + + leave_nodes = self._collect_leave_nodes_for_evict() + heapq.heapify(leave_nodes) + evicted_indices: List[torch.Tensor] = [] + evicted_size = 0 + + while evicted_size < size: + assert ( + leave_nodes + ), f"Cannot evict enough cache, need {size}, only {evicted_size} evicted" + node = heapq.heappop(leave_nodes) + assert node.ref_count == 0 and node.is_leaf() and not node.is_root() + evicted_size += node.length + evicted_indices.append(node.value) + self.evictable_size -= node.length + parent = node.parent + del parent.children[int(node._key[0].item())] + # NOTE: root is always protected, so won't be evicted + if parent.is_leaf() and parent.ref_count == 0: + heapq.heappush(leave_nodes, parent) + + return torch.cat(evicted_indices) + + def _collect_leave_nodes_for_evict(self) -> List[RadixTreeNode]: + nodes: List[RadixTreeNode] = [self.root_node] + leave_nodes: List[RadixTreeNode] = [] + + while len(nodes) > 0: + node = nodes.pop() + if node.is_leaf(): + if node.ref_count == 0: + leave_nodes.append(node) + else: + for child in node.children.values(): + nodes.append(child) + + return leave_nodes + + def reset(self) -> None: + raise NotImplementedError("RadixManager.reset is not implemented") + + @property + def size_info(self) -> SizeInfo: + return SizeInfo( + evictable_size=self.evictable_size, + protected_size=self.protected_size, + ) + + def check_integrity(self) -> None: + pass diff --git a/zonos2/layers/__init__.py b/zonos2/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f2a1efe60069cdd5cb0b2b3485c888da8c595754 --- /dev/null +++ b/zonos2/layers/__init__.py @@ -0,0 +1,32 @@ +from .activation import silu_and_mul +from .attention import AttentionLayer +from .base import BaseOP, OPList, StateLessOP +from .embedding import ParallelLMHead, VocabParallelEmbedding +from .linear import ( + ChunkedLinear, + LinearColParallelMerged, + LinearOProj, + LinearQKVMerged, + LinearRowParallel, +) +from .norm import RMSNorm, RMSNormFused +from .rotary import get_rope, set_rope_device + +__all__ = [ + "silu_and_mul", + "AttentionLayer", + "BaseOP", + "StateLessOP", + "OPList", + "VocabParallelEmbedding", + "ParallelLMHead", + "ChunkedLinear", + "LinearColParallelMerged", + "LinearRowParallel", + "LinearOProj", + "LinearQKVMerged", + "RMSNorm", + "RMSNormFused", + "get_rope", + "set_rope_device", +] diff --git a/zonos2/layers/activation.py b/zonos2/layers/activation.py new file mode 100644 index 0000000000000000000000000000000000000000..3de064c401ea9e982fee2b8d3b1dd14c61337863 --- /dev/null +++ b/zonos2/layers/activation.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import torch + + +def silu_and_mul(x: torch.Tensor) -> torch.Tensor: + from flashinfer import silu_and_mul + + return silu_and_mul(x) + + +__all__ = ["silu_and_mul"] diff --git a/zonos2/layers/attention.py b/zonos2/layers/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c88df8918517cd30079610d7df425ac1de4d9e --- /dev/null +++ b/zonos2/layers/attention.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from zonos2.core import get_global_ctx +from zonos2.distributed import get_tp_info +from zonos2.utils import divide_even + +from .base import StateLessOP +from .rotary import get_rope + +if TYPE_CHECKING: + from zonos2.layers import RMSNorm + from zonos2.models import RotaryConfig + + +class AttentionLayer(StateLessOP): + def __init__( + self, + layer_id: int, + num_qo_heads: int, + num_kv_heads: int, + head_dim: int, + rotary_config: RotaryConfig, + q_norm: RMSNorm | None = None, + k_norm: RMSNorm | None = None, + ): + assert num_qo_heads % num_kv_heads == 0 + self.layer_id = layer_id + self.head_dim = head_dim + tp_size = get_tp_info().size + self.num_qo_heads = divide_even(num_qo_heads, tp_size) + self.num_kv_heads = divide_even(num_kv_heads, tp_size) + self.qo_attn_dim = self.num_qo_heads * head_dim + self.kv_attn_dim = self.num_kv_heads * head_dim + self.rotary = get_rope( + head_dim=head_dim, + rotary_dim=rotary_config.rotary_dim, + max_position=rotary_config.max_position, + base=rotary_config.base, + rope_scaling=tuple(rotary_config.scaling.items()) if rotary_config.scaling else None, + ) + self.q_norm = q_norm + self.k_norm = k_norm + + def forward(self, qkv: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + metadata = ctx.batch.attn_metadata + q, k, v = qkv.split([self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim], dim=-1) + if self.q_norm is not None: + self.q_norm.forward_inplace(q.view(-1, self.num_qo_heads, self.head_dim)) + if self.k_norm is not None: + self.k_norm.forward_inplace(k.view(-1, self.num_kv_heads, self.head_dim)) + if self.rotary: + q, k = self.rotary.forward(metadata.positions, q, k) + q = q.view(-1, self.num_qo_heads, self.head_dim) + o = ctx.attn_backend.forward(q, k, v, self.layer_id, ctx.batch) + return o.view(-1, self.qo_attn_dim) diff --git a/zonos2/layers/base.py b/zonos2/layers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..4c0fb76a39dbb7a0efe8085ab72e3ea14ea3c106 --- /dev/null +++ b/zonos2/layers/base.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import re +from abc import abstractmethod +from typing import Any, Dict, Generic, List, TypeAlias, TypeVar + +import torch + +_STATE_DICT: TypeAlias = Dict[str, torch.Tensor] + + +def _concat_prefix(prefix: str, name: str) -> str: + return f"{prefix}.{name}" if prefix else name + + +class BaseOP: + @abstractmethod + def forward(self, *args: Any, **kwargs: Any) -> Any: ... + + def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT: + result = result if result is not None else {} + + for name, param in self.__dict__.items(): + if name.startswith("_"): + continue + if isinstance(param, torch.Tensor): + result[_concat_prefix(prefix, name)] = param + elif isinstance(param, BaseOP): + param.state_dict(prefix=_concat_prefix(prefix, name), result=result) + + return result + + def load_state_dict( + self, + state_dict: _STATE_DICT, + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + for name, param in self.__dict__.items(): + if name.startswith("_"): + continue + + if isinstance(param, torch.Tensor): + if "experts" in prefix: + mapped_name = name + matched_keys = [] + for key in list(state_dict.keys()): + if prefix in key and mapped_name in key: + matched_keys.append(key) + + def extract_expert_index(k): + match = re.search(r"experts\.(\d+)\.", k) + return int(match.group(1)) if match else -1 + + matched_keys.sort(key=extract_expert_index) + + if not matched_keys: + raise ValueError( + f"No weights found in state_dict for {prefix} and {mapped_name}" + ) + + # Check if weights are already stacked (no per-expert indices) + # or per-expert (have indices like experts.0.gate_up_proj) + has_expert_indices = any( + re.search(r"experts\.\d+\.", k) for k in matched_keys + ) + + if has_expert_indices: + # Per-expert weights - stack them + items = [] + for k in matched_keys: + items.append(state_dict.pop(k)) + item = torch.stack(items, dim=0) + else: + # Already stacked weights - use directly + assert len(matched_keys) == 1, ( + f"Expected single stacked weight key, got {matched_keys}" + ) + item = state_dict.pop(matched_keys[0]) + else: + item = state_dict.pop(_concat_prefix(prefix, name)) + + assert isinstance(item, torch.Tensor) + assert ( + param.shape == item.shape + ), f"Shape mismatch: param {param.shape} vs item {item.shape}" + if param.dtype != item.dtype: + if param.is_floating_point() and item.is_floating_point(): + item = item.to(dtype=param.dtype) + else: + key = _concat_prefix(prefix, name) + raise AssertionError( + f"Dtype mismatch for {key}: param {param.dtype} vs item {item.dtype}" + ) + + setattr(self, name, item) + + elif isinstance(param, BaseOP): + param.load_state_dict( + state_dict, prefix=_concat_prefix(prefix, name), _internal=True + ) + + if not _internal and state_dict: + keys = list(state_dict.keys()) + raise RuntimeError( + f"Unexpected keys in state_dict: {len(keys)} keys (first 10: {keys[:10]})" + ) + + +class StateLessOP(BaseOP): + def __init__(self): + super().__init__() + + def load_state_dict( + self, + state_dict: _STATE_DICT, + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + if not _internal and state_dict: + _ = prefix + keys = list(state_dict.keys()) + raise RuntimeError( + f"Unexpected keys in state_dict: {len(keys)} keys (first 10: {keys[:10]})" + ) + + def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT: + _ = prefix + return result if result is not None else {} + + +T = TypeVar("T", bound=BaseOP) + + +class OPList(BaseOP, Generic[T]): + def __init__(self, ops: List[T]): + super().__init__() + self.op_list = ops + + def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT: + result = result if result is not None else {} + for i, op in enumerate(self.op_list): + op.state_dict(prefix=_concat_prefix(prefix, str(i)), result=result) + return result + + def load_state_dict( + self, + state_dict: _STATE_DICT, + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + for i, op in enumerate(self.op_list): + op.load_state_dict(state_dict, prefix=_concat_prefix(prefix, str(i)), _internal=True) + + if not _internal and state_dict: + keys = list(state_dict.keys()) + raise RuntimeError( + f"Unexpected keys in state_dict: {len(keys)} keys (first 10: {keys[:10]})" + ) diff --git a/zonos2/layers/embedding.py b/zonos2/layers/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..ff695eea21d92004cfd89bd0af6672f3e819d846 --- /dev/null +++ b/zonos2/layers/embedding.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import Dict + +import torch +import torch.nn.functional as F +from zonos2.core import get_global_ctx +from zonos2.distributed import DistributedCommunicator, get_tp_info +from zonos2.utils import divide_up, nvtx_annotate + +from .base import BaseOP + + +class VocabParallelEmbedding(BaseOP): + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + ): + super().__init__() + tp_info = get_tp_info() + tp_rank = tp_info.rank + self.tp_size = tp_info.size + self.num_embeddings = num_embeddings + self.num_embeddings_tp = divide_up(num_embeddings, self.tp_size) + start_idx = self.num_embeddings_tp * tp_rank + finish_idx = min(start_idx + self.num_embeddings_tp, num_embeddings) + self.vocab_range = (start_idx, finish_idx - start_idx) + self.weight = torch.empty(self.num_embeddings_tp, embedding_dim) + self._comm = DistributedCommunicator() + + @nvtx_annotate("Embedding") + def forward(self, x: torch.Tensor) -> torch.Tensor: + from zonos2.kernel import indexing + + y = indexing( + weights=self.weight, + indices=x, + vocab_range=self.vocab_range if self.tp_size > 1 else None, + ) + + return self._comm.all_reduce(y) if self.tp_size > 1 else y + + +class ParallelLMHead(VocabParallelEmbedding): + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + bias: bool = False, + tie_word_embeddings: bool = False, + tied_embedding: VocabParallelEmbedding | None = None, + ): + super().__init__(num_embeddings, embedding_dim) + self.bias = torch.empty(self.num_embeddings_tp) if bias else None + self.tied_embedding = tied_embedding + assert (tied_embedding is not None) == tie_word_embeddings + + def load_state_dict( + self, + state_dict: Dict[str, torch.Tensor], + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + if not self.tied_embedding: + return super().load_state_dict(state_dict, prefix=prefix, _internal=_internal) + else: + # pop the lm_head.weights and lm_head.bias if they exist + possible_weight = f"{prefix}.weight" + possible_bias = f"{prefix}.bias" + if possible_weight in state_dict: + state_dict.pop(possible_weight) + if possible_bias in state_dict: + state_dict.pop(possible_bias) + + def state_dict( + self, + *, + prefix: str = "", + result: Dict[str, torch.Tensor] | None = None, + ) -> Dict[str, torch.Tensor]: + if not self.tied_embedding: + return super().state_dict(prefix=prefix, result=result) + return {} if result is None else result + + @nvtx_annotate("LMHead") + def forward(self, x: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + batch = ctx.batch + bs = batch.size + if batch.is_prefill: + indices = batch.attn_metadata.get_last_indices(bs) + x = x[indices].contiguous() + del indices + + module = self.tied_embedding or self + logits = F.linear(x, module.weight, self.bias) + if self.tp_size == 1: + return logits + input_shape = logits.shape + output_tensor = self._comm.all_gather(logits) + + if bs == 1: + return output_tensor.view(1, -1)[:, : self.num_embeddings] + + output_tensor = output_tensor.view((self.tp_size,) + input_shape) + output_tensor = output_tensor.movedim(0, -1) + output_tensor = output_tensor.reshape(input_shape[:1] + (self.tp_size * input_shape[1],)) + return output_tensor[:, : self.num_embeddings] diff --git a/zonos2/layers/linear.py b/zonos2/layers/linear.py new file mode 100644 index 0000000000000000000000000000000000000000..9661a3061f147a5590668aef52d5e9c10d344fa5 --- /dev/null +++ b/zonos2/layers/linear.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from typing import List + +import torch +import torch.nn.functional as F +from zonos2.distributed import DistributedCommunicator, get_tp_info +from zonos2.utils import divide_even + +from .base import BaseOP + + +class _LinearTPImpl(BaseOP): + """Real implementation of a linear layer with tensor parallelism.""" + + def __init__( + self, + full_isize: int, + full_osize: int, + local_isize: int, + local_osize: int, + has_bias: bool, + ): + self.full_input_size = full_isize + self.full_output_size = full_osize + self.local_input_size = local_isize + self.local_output_size = local_osize + self.weight = torch.empty(local_osize, local_isize) + self.bias = torch.empty(local_osize) if has_bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) + + +class LinearReplicated(_LinearTPImpl): + """ + Linear layer where weights are replicated (not sharded) across all TP ranks. + Each GPU holds the full weight matrix. + """ + + def __init__( + self, + input_size: int, + output_size: int, + has_bias: bool, + ): + super().__init__( + full_isize=input_size, + full_osize=output_size, + local_isize=input_size, + local_osize=output_size, + has_bias=has_bias, + ) + + +class LinearColParallelMerged(_LinearTPImpl): + def __init__( + self, + input_size: int, + output_sizes: List[int], + has_bias: bool, + ): + # check that all output sizes are divisible by tp_size + tp_info = get_tp_info() + tp_output_sizes = [divide_even(size, tp_info.size) for size in output_sizes] + output_size = sum(output_sizes) + tp_output_size = sum(tp_output_sizes) + super().__init__(input_size, output_size, input_size, tp_output_size, has_bias) + + +class LinearQKVMerged(_LinearTPImpl): + def __init__( + self, + hidden_size: int, + head_dim: int, + num_qo_heads: int, + num_kv_heads: int, + has_bias: bool, + ): + tp_info = get_tp_info() + + GQA_ratio = divide_even(num_qo_heads, num_kv_heads) + local_num_kv = divide_even(num_kv_heads, tp_info.size) + full_isize = hidden_size + full_osize = (GQA_ratio + 2) * num_kv_heads * head_dim + local_isize = hidden_size + local_osize = (GQA_ratio + 2) * local_num_kv * head_dim + super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias) + + +class LinearOProj(_LinearTPImpl): + def __init__(self, input_size: int, output_size: int, has_bias: bool): + tp_info = get_tp_info() + full_isize = input_size + full_osize = output_size + local_isize = divide_even(input_size, tp_info.size) + local_osize = output_size + self._comm = DistributedCommunicator() + self._tp_size = tp_info.size + super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = F.linear(x, self.weight, self.bias) + if self._tp_size > 1: + y = self._comm.all_reduce(y) + return y + + +class LinearRowParallel(_LinearTPImpl): + def __init__( + self, + input_size: int, + output_size: int, + has_bias: bool, + ): + tp_info = get_tp_info() + local_input_size = divide_even(input_size, tp_info.size) + local_output_size = output_size + self._comm = DistributedCommunicator() + self._tp_size = tp_info.size + super().__init__(input_size, output_size, local_input_size, local_output_size, has_bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = F.linear(x, self.weight, self.bias) + if self._tp_size > 1: + y = self._comm.all_reduce(y) + return y + + +class ChunkedLinear(BaseOP): + """Linear layer with 3D weight [divisor, out_per_chunk, in_features]. + + Used for fused projections (e.g., wkv for K/V, w_in for gate/up). + The weight is stored as 3D but reshaped to 2D for the linear operation. + """ + + def __init__(self, in_features: int, out_features: int, divisor: int, has_bias: bool = False): + if out_features % divisor != 0: + raise ValueError(f"out_features ({out_features}) must be divisible by divisor ({divisor}).") + self.in_features = int(in_features) + self.out_features = int(out_features) + self.divisor = int(divisor) + self.out_per_chunk = int(out_features // divisor) + # 3D weight: [divisor, out_per_chunk, in_features] + self.weight = torch.empty(self.divisor, self.out_per_chunk, self.in_features) + self.bias = torch.empty(self.out_features) if has_bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Reshape 3D weight to 2D for standard linear operation + w2d = self.weight.view(self.out_features, self.in_features) + return F.linear(x, w2d, self.bias) diff --git a/zonos2/layers/moe/fused_moe/fused_moe_impl.py b/zonos2/layers/moe/fused_moe/fused_moe_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..0367f1db2674eced521d0ef9df1131981f2918e7 --- /dev/null +++ b/zonos2/layers/moe/fused_moe/fused_moe_impl.py @@ -0,0 +1,413 @@ +import functools +from typing import Dict, Optional, Tuple + +import torch +import triton +import triton.language as tl +from sgl_kernel import gelu_and_mul, silu_and_mul +from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size +from zonos2.kernel.moe_impl import fused_moe_kernel_triton +from zonos2.kernel.triton.fused_moe import moe_sum_reduce_triton +from zonos2.layers.moe.fused_moe.topk import select_experts + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def moe_sum_reduce_torch_compile(x, out, routed_scaling_factor): + torch.sum(x, dim=1, out=out) + out.mul_(routed_scaling_factor) + + +def is_cuda(): + return torch.cuda.is_available() and torch.version.cuda + + +def moe_align_block_size( + topk_ids: torch.Tensor, block_size: int, num_experts: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Aligns the token distribution across experts to be compatible with block + size for matrix multiplication. + + Parameters: + - topk_ids: A tensor of shape [total_tokens, top_k] representing the + top-k expert indices for each token. + - block_size: The block size used in block matrix multiplication. + - num_experts: The total number of experts. + + Returns: + - sorted_token_ids: A tensor containing the sorted token indices according + to their allocated expert. + - expert_ids: A tensor indicating the assigned expert index for each block. + - num_tokens_post_padded: The total number of tokens after padding, + ensuring divisibility by block_size. + + This function pads the number of tokens that each expert needs to process + so that it is divisible by block_size. + Padding ensures that during block matrix multiplication, the dimensions + align correctly. + + Example: + Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], + block_size = 4, and num_experts = 4: + - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, + with each expert needing to process 3 tokens. + - As block_size is 4, we pad 1 token for each expert. + - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3]. + - Then append padding tokens [12, 12, 12, 12] for each block. + - After sorting by expert index, we obtain token_ids + [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12]. + Tokens 12 are non-existent (padding) and are ignored in + the subsequent matrix multiplication. + - The padding ensures that the total number of tokens is now divisible + by block_size for proper block matrix operations. + """ + max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device) + max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size) + expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device) + num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device) + + cumsum_buffer = torch.empty((num_experts + 2,), dtype=torch.int32, device=topk_ids.device) + + sgl_moe_align_block_size( + topk_ids, + num_experts + 1, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + cumsum_buffer, + True, + ) + return sorted_ids, expert_ids, num_tokens_post_pad + + +def get_default_config( + M: int, + E: int, + N: int, + K: int, + topk: int, + is_marlin: bool, +) -> Dict[str, int]: + + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + } + # A heuristic: fused marlin works faster with this config for small M + if M <= E or (is_marlin and M <= 32): + config = { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + } + return config + + +def try_get_optimal_moe_config( + w1_shape: Tuple[int, ...], + w2_shape: Tuple[int, ...], + top_k: int, + M: int, + is_marlin: bool = False, +): + E, _, N = w2_shape + + config = get_default_config(M, E, N, w1_shape[2], top_k, is_marlin) + return config + + +def fused_experts_impl( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + inplace: bool = False, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, +): + + padded_size = 0 + assert hidden_states.shape[1] == w1.shape[2] - padded_size, "Hidden size mismatch" + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.is_contiguous(), "Expert weights1 must be contiguous" + assert w2.is_contiguous(), "Expert weights2 must be contiguous" + assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16] + num_tokens, _ = hidden_states.shape + E, N, _ = w1.shape + + CHUNK_SIZE = 64 * 1024 + M = min(num_tokens, CHUNK_SIZE) + + get_config_func = functools.partial( + try_get_optimal_moe_config, + w1.shape, + (w2.shape[0], w2.shape[1], w2.shape[2] - padded_size), + topk_ids.shape[1], + ) + config = get_config_func(M) + + cache = torch.empty( + M * topk_ids.shape[1] * max(N, w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache1 = cache[: M * topk_ids.shape[1] * N].view( + (M, topk_ids.shape[1], N), + ) + intermediate_cache2 = torch.empty( + (M * topk_ids.shape[1], N // 2), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache3 = cache[: M * topk_ids.shape[1] * w2.shape[1]].view( + (M, topk_ids.shape[1], w2.shape[1]), + ) + + compute_type = tl.bfloat16 if hidden_states.dtype == torch.bfloat16 else tl.float16 + if no_combine: + assert not inplace + out_hidden_states = torch.empty( + (num_tokens, topk_ids.shape[1], w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + elif inplace: + out_hidden_states = hidden_states + else: + out_hidden_states = torch.empty_like(hidden_states) + + for chunk in range((num_tokens // CHUNK_SIZE) + 1): + begin_chunk_idx, end_chunk_idx = ( + chunk * CHUNK_SIZE, + min((chunk + 1) * CHUNK_SIZE, num_tokens), + ) + curr_hidden_states = hidden_states[begin_chunk_idx:end_chunk_idx] + tokens_in_chunk, _ = curr_hidden_states.shape + + if tokens_in_chunk == 0: + break + + if tokens_in_chunk < CHUNK_SIZE and chunk > 0: + # Adjust the intermediate cache size and config for the last + # chunk. Note that in most cases we only have one chunk + # so the cache size and config are already set correctly and + # do not need to be adjusted. + intermediate_cache1 = intermediate_cache1[:tokens_in_chunk] + intermediate_cache2 = intermediate_cache2[: tokens_in_chunk * topk_ids.shape[1]] + intermediate_cache3 = intermediate_cache3[:tokens_in_chunk] + config = get_config_func(tokens_in_chunk) + + curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx] + curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx] + + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + curr_topk_ids, config["BLOCK_SIZE_M"], E + ) + + fused_moe_kernel_triton( + curr_hidden_states, + w1, + intermediate_cache1, + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + apply_router_weight_on_input, + topk_ids.shape[1], + config, + compute_type=compute_type, + ) + + if activation == "silu": + + silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + + elif activation == "gelu": + + gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + + else: + raise ValueError(f"Unsupported activation: {activation=}") + + fused_moe_kernel_triton( + intermediate_cache2, + w2, + ( + intermediate_cache3 + if not no_combine and topk_ids.shape[1] != 1 + else out_hidden_states[begin_chunk_idx:end_chunk_idx].unsqueeze(0) + ), + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + config, + compute_type=compute_type, + ) + + if routed_scaling_factor is None: + routed_scaling_factor = 1.0 + + if no_combine: + pass + + if topk_ids.shape[1] == 1 and routed_scaling_factor == 1.0: + pass # we write directly into out_hidden_states + elif topk_ids.shape[1] == 2 and routed_scaling_factor == 1.0: + torch.add( + intermediate_cache3[:, 0], + intermediate_cache3[:, 1], + out=out_hidden_states[begin_chunk_idx:end_chunk_idx], + ).squeeze(dim=1) + else: + # According to micro benchmark results, torch.compile can get better performance for small token. + if tokens_in_chunk <= 32: + + moe_sum_reduce_torch_compile( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + else: + moe_sum_reduce_triton( + intermediate_cache3, + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + return out_hidden_states + + +def inplace_fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + routed_scaling_factor: Optional[float] = None, +) -> None: + + fused_experts_impl( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + True, + activation, + apply_router_weight_on_input, + False, + routed_scaling_factor, + ) + + +def outplace_fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, +) -> torch.Tensor: + return fused_experts_impl( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + False, + activation, + apply_router_weight_on_input, + no_combine=no_combine, + routed_scaling_factor=routed_scaling_factor, + ) + + +def fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + inplace: bool = False, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, +): + + if inplace: + assert not no_combine, "no combine + inplace makes no sense" + inplace_fused_experts( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + activation, + apply_router_weight_on_input, + routed_scaling_factor, + ) + return hidden_states + else: + return outplace_fused_experts( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + activation, + apply_router_weight_on_input, + no_combine=no_combine, + routed_scaling_factor=routed_scaling_factor, + ) + + +def fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + inplace: bool = False, + activation: str = "silu", + no_combine: bool = False, +) -> torch.Tensor: + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=gating_output, + top_k=topk, + renormalize=renormalize, + ) + return fused_experts( + hidden_states=hidden_states, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=inplace, + activation=activation, + no_combine=no_combine, + ) diff --git a/zonos2/layers/moe/fused_moe/layer.py b/zonos2/layers/moe/fused_moe/layer.py new file mode 100644 index 0000000000000000000000000000000000000000..cad9f139cee2d0a6c301bd44ab02bf197b9b302e --- /dev/null +++ b/zonos2/layers/moe/fused_moe/layer.py @@ -0,0 +1,178 @@ +from typing import List, Optional + +import torch +import torch.nn as nn +from zonos2.distributed import DistributedCommunicator, get_tp_info +from zonos2.layers.base import BaseOP +from zonos2.layers.moe.fused_moe.fused_moe_impl import fused_experts, fused_moe +from zonos2.utils import divide_even + + +class FusedMoEMethod(nn.Module): + def __init__(self): + super().__init__() + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size: int, + params_dtype: torch.dtype, + tp_size: int = 1, + tp_rank: int = 0, + **extra_weight_attrs, + ): + intermediate_size_per_partition = divide_even(intermediate_size, tp_size) + + gate_up_proj = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + setattr(layer, "gate_up_proj", gate_up_proj) + + down_proj = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + setattr(layer, "down_proj", down_proj) + + def forward( + self, + layer: torch.nn.Module, + x: torch.Tensor, + top_k: int, + router_logits: torch.Tensor, + renormalize: bool, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + block_shape: Optional[List[int]] = None, + inplace: bool = True, + no_combine: bool = False, + ) -> torch.Tensor: + return fused_moe( + hidden_states=x, + w1=layer.gate_up_proj, + w2=layer.down_proj, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + inplace=inplace, + activation=activation, + no_combine=no_combine, + ) + + +class FusedMoE(BaseOP): + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + layer_id: Optional[int] = None, + params_dtype: Optional[torch.dtype] = None, + renormalize: bool = True, + tp_size: Optional[int] = None, + prefix: str = "", + activation: str = "silu", + apply_router_weight_on_input: bool = False, + inplace: bool = True, + no_combine: bool = False, + ): + super().__init__() + if params_dtype is None: + params_dtype = torch.get_default_dtype() + + self.num_experts = num_experts + self.top_k = top_k + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.params_dtype = params_dtype + self._comm = DistributedCommunicator() + + tp_info = get_tp_info() + self.tp_size = tp_info.size if tp_size is None else tp_size + self.tp_rank = tp_info.rank if tp_size is None else 0 + + assert self.intermediate_size % self.tp_size == 0, ( + f"Intermediate size ({self.intermediate_size}) must be divisible " + f"by tp_size ({self.tp_size})" + ) + + self.intermediate_size_per_partition = self.intermediate_size // self.tp_size + self.renormalize = renormalize + self.activation = activation + self.apply_router_weight_on_input = apply_router_weight_on_input + self.inplace = inplace + self.no_combine = no_combine + self.layer_id = layer_id + self.prefix = prefix + + self.fused_moe_method: Optional[FusedMoEMethod] = FusedMoEMethod() + self.fused_moe_method.create_weights( + self, + num_experts=num_experts, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + params_dtype=params_dtype, + tp_size=self.tp_size, + tp_rank=self.tp_rank, + ) + + def forward( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor | None = None, + *, + topk_weights: torch.Tensor | None = None, + topk_ids: torch.Tensor | None = None, + ): + assert self.fused_moe_method is not None + + if topk_weights is None and topk_ids is None: + if router_logits is None: + raise ValueError("router_logits must be provided when top-k routing is not precomputed.") + final_hidden_states = self.fused_moe_method.forward( + layer=self, + x=hidden_states, + router_logits=router_logits, + top_k=self.top_k, + renormalize=self.renormalize, + activation=self.activation, + apply_router_weight_on_input=self.apply_router_weight_on_input, + inplace=self.inplace, + no_combine=self.no_combine, + ) + else: + if topk_weights is None or topk_ids is None: + raise ValueError("topk_weights and topk_ids must be provided together.") + topk_weights = topk_weights.to(dtype=torch.float32) + if self.renormalize: + topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-8) + final_hidden_states = fused_experts( + hidden_states=hidden_states, + w1=self.gate_up_proj, + w2=self.down_proj, + topk_weights=topk_weights.contiguous(), + topk_ids=topk_ids.to(dtype=torch.int32).contiguous(), + inplace=self.inplace, + activation=self.activation, + apply_router_weight_on_input=self.apply_router_weight_on_input, + no_combine=self.no_combine, + ) + + if self.tp_size > 1: + final_hidden_states = self._comm.all_reduce(final_hidden_states) + + return final_hidden_states diff --git a/zonos2/layers/moe/fused_moe/topk.py b/zonos2/layers/moe/fused_moe/topk.py new file mode 100644 index 0000000000000000000000000000000000000000..1f18ea8b6ad879c69df13ff1fb155d86bb9d286a --- /dev/null +++ b/zonos2/layers/moe/fused_moe/topk.py @@ -0,0 +1,80 @@ +from typing import Any, Optional + +import torch + +try: + from sgl_kernel import topk_softmax +except ImportError: + + def topk_softmax(*args: Any, **kwargs: Any) -> Any: + raise NotImplementedError("sgl_kernel not found.") + + +def fused_topk( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_token_non_padded: Optional[torch.Tensor] = None, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + + M, _ = hidden_states.shape + + topk_weights = torch.empty(M, topk, dtype=torch.float32, device=hidden_states.device) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + + topk_softmax( + topk_weights, + topk_ids, + gating_output.float(), + renormalize, + ) + + return _fused_topk_postprocess( + topk_weights=topk_weights, + topk_ids=topk_ids, + renormalize=renormalize, + num_token_non_padded=num_token_non_padded, + ) + + +def _mask_topk_ids_padded_region( + topk_ids: torch.Tensor, + num_token_non_padded: Optional[torch.Tensor] = None, +): + if num_token_non_padded is None: + return + + indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device) + topk_ids[indices >= num_token_non_padded, :] = -1 + + +def _fused_topk_postprocess( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + renormalize: bool, + num_token_non_padded: Optional[torch.Tensor], +): + if renormalize: + topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-8) + + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_weights, topk_ids + + +def select_experts( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + num_token_non_padded: Optional[torch.Tensor] = None, + renormalize: bool = True, +): + topk_weights, topk_ids = fused_topk( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + num_token_non_padded=num_token_non_padded, + ) + return topk_weights, topk_ids diff --git a/zonos2/layers/moe/moe_backend.py b/zonos2/layers/moe/moe_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..e92380c846c24f47194e132b8ba57d79a463f474 --- /dev/null +++ b/zonos2/layers/moe/moe_backend.py @@ -0,0 +1,16 @@ +from zonos2.layers.moe.fused_moe.layer import FusedMoE +from zonos2.models import ModelConfig + + +def get_moe_backend(moe_backend: str, config: ModelConfig, prefix: str = ""): + if moe_backend == "fused_moe": + return FusedMoE( + num_experts=config.num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + renormalize=config.norm_topk_prob, + prefix=prefix, + ) + else: + raise ValueError(f"Unsupported MoE backend: {moe_backend}") diff --git a/zonos2/layers/norm.py b/zonos2/layers/norm.py new file mode 100644 index 0000000000000000000000000000000000000000..6afafd4ed235142c9c592daea2bffdbe33e840b7 --- /dev/null +++ b/zonos2/layers/norm.py @@ -0,0 +1,55 @@ +from typing import Tuple + +import torch + +from .base import BaseOP + + +class RMSNorm(BaseOP): + def __init__(self, size: int, eps: float) -> None: + from flashinfer import rmsnorm + + self.eps = eps + self.weight = torch.empty(size) + self.rmsnorm = rmsnorm + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.rmsnorm(x, self.weight, self.eps) + + def forward_inplace(self, x: torch.Tensor) -> None: + self.rmsnorm(x, self.weight, self.eps, out=x) + + +class RMSNormFused(BaseOP): + def __init__(self, size: int, eps: float, elementwise_affine: bool = True) -> None: + from flashinfer import fused_add_rmsnorm, rmsnorm + + self.eps = eps + self.elementwise_affine = elementwise_affine + self._size = size + + if elementwise_affine: + self.weight = torch.empty(size) + # When elementwise_affine=False, we use a ones buffer created lazily + # to ensure correct device/dtype + + self.rmsnorm = rmsnorm + self.fused_add_rmsnorm = fused_add_rmsnorm + self._ones_buffer: torch.Tensor | None = None + + def _get_weight(self, x: torch.Tensor) -> torch.Tensor: + if self.elementwise_affine: + return self.weight + # Use cached ones buffer, recreate if needed for device/dtype match + if self._ones_buffer is None or self._ones_buffer.device != x.device: + self._ones_buffer = torch.ones(self._size, device=x.device, dtype=x.dtype) + return self._ones_buffer + + def forward( + self, x: torch.Tensor, residual: torch.Tensor | None = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + weight = self._get_weight(x) + if residual is None: + return self.rmsnorm(x, weight, self.eps), x + self.fused_add_rmsnorm(x, residual, weight, self.eps) + return x, residual diff --git a/zonos2/layers/rotary.py b/zonos2/layers/rotary.py new file mode 100644 index 0000000000000000000000000000000000000000..9b92c21905a9dbe4a4bd5b0b788bc9743aba6f95 --- /dev/null +++ b/zonos2/layers/rotary.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import Any, Callable, Dict, Tuple + +import torch + +from .base import StateLessOP + + +class RotaryEmbedding(StateLessOP): + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: float, + post_process: None | Callable[[torch.Tensor], torch.Tensor] = None, + ) -> None: + super().__init__() + self.head_size = head_size + assert rotary_dim == head_size + inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim)) + if post_process is not None: + inv_freq = post_process(inv_freq) + t = torch.arange(max_position_embeddings, dtype=torch.float) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + # buffer, so don't load/save + self._cos_sin_cache = torch.cat((cos, sin), dim=-1) + assert self.head_size in [64, 128, 256, 512] + + from flashinfer import apply_rope_with_cos_sin_cache_inplace + + self.apply_rope_with_cos_sin_cache_inplace = apply_rope_with_cos_sin_cache_inplace + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + is_neox: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor]: + self.apply_rope_with_cos_sin_cache_inplace( + positions=positions, + query=query, + key=key, + head_size=self.head_size, + cos_sin_cache=self._cos_sin_cache, + is_neox=is_neox, + ) + return query, key + + +def _get_rope( + head_dim: int, + rotary_dim: int, + max_position: int, + base: float, + rope_scaling: Dict[str, Any] | None = None, +) -> RotaryEmbedding: + if rope_scaling is None: + return RotaryEmbedding(head_dim, rotary_dim, max_position, base) + raise ValueError(f"Unsupported rotary scaling in TTS release: {rope_scaling!r}") + + +_ROPE_DEVICE: torch.device | None = None + + +def set_rope_device(device: torch.device): + global _ROPE_DEVICE + _ROPE_DEVICE = device + + +@lru_cache() +def get_rope( + head_dim: int, + rotary_dim: int, + max_position: int, + base: float, + rope_scaling: Tuple[Tuple[str, Any], ...] | None = None, +) -> RotaryEmbedding: + rope_map = dict(rope_scaling) if rope_scaling is not None else None + t = torch.tensor([]) + if t.device == torch.device("meta"): + # we cannot use meta device for rope + if _ROPE_DEVICE is None: + raise RuntimeError( + "We cannot use meta device for rope. Please call set_rope_device() first." + ) + with torch.device(_ROPE_DEVICE): + return _get_rope(head_dim, rotary_dim, max_position, base, rope_map) + return _get_rope(head_dim, rotary_dim, max_position, base, rope_map) + + +__all__ = ["get_rope", "RotaryEmbedding", "set_rope_device"] diff --git a/zonos2/message/__init__.py b/zonos2/message/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c1bf2aee2e28ff421cfed75e7fa91400aca9395 --- /dev/null +++ b/zonos2/message/__init__.py @@ -0,0 +1,36 @@ +from .backend import BaseBackendMsg, ExitMsg +from .frontend import BaseFrontendMsg +from .tokenizer import BaseTokenizerMsg, BatchTokenizerMsg +from .tts import ( + BaseTTSBackendMsg, + BaseTTSFrontendMsg, + BaseTTSTokenizerMsg, + BatchTTSBackendMsg, + BatchTTSFrontendMsg, + BatchTTSTokenizerMsg, + TTSAudioReply, + TTSDetokenizeMsg, + TTSSamplingParams, + TTSTokenizeMsg, + TTSUserMsg, +) + +__all__ = [ + "BaseBackendMsg", + "ExitMsg", + "BaseTokenizerMsg", + "BatchTokenizerMsg", + "BaseFrontendMsg", + # TTS message types + "BaseTTSBackendMsg", + "BaseTTSFrontendMsg", + "BaseTTSTokenizerMsg", + "BatchTTSBackendMsg", + "BatchTTSFrontendMsg", + "BatchTTSTokenizerMsg", + "TTSAudioReply", + "TTSDetokenizeMsg", + "TTSSamplingParams", + "TTSTokenizeMsg", + "TTSUserMsg", +] diff --git a/zonos2/message/backend.py b/zonos2/message/backend.py new file mode 100644 index 0000000000000000000000000000000000000000..14798907e86d91fef298601a2b14c06565c9cbc8 --- /dev/null +++ b/zonos2/message/backend.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict + +# Import TTS message classes so they're available in globals() for deserialization +from .tts import ( # noqa: F401 + BatchTTSBackendMsg, + TTSSamplingParams, + TTSUserMsg, +) +from .utils import deserialize_type, serialize_type + + +@dataclass +class BaseBackendMsg: + def encoder(self) -> Dict: + return serialize_type(self) + + @staticmethod + def decoder(json: Dict) -> BaseBackendMsg: + return deserialize_type(globals(), json) + + +@dataclass +class ExitMsg(BaseBackendMsg): + pass diff --git a/zonos2/message/frontend.py b/zonos2/message/frontend.py new file mode 100644 index 0000000000000000000000000000000000000000..f93453ffc2b9722fe6744f89d8528e8035524321 --- /dev/null +++ b/zonos2/message/frontend.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict + +# Import TTS message classes so they're available in globals() for deserialization +from .tts import ( # noqa: F401 + BatchTTSFrontendMsg, + TTSAudioReply, +) +from .utils import deserialize_type, serialize_type + + +@dataclass +class BaseFrontendMsg: + @staticmethod + def encoder(msg: BaseFrontendMsg) -> Dict: + return serialize_type(msg) + + @staticmethod + def decoder(json: Dict) -> BaseFrontendMsg: + return deserialize_type(globals(), json) + diff --git a/zonos2/message/tokenizer.py b/zonos2/message/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..262fb573bc78d36da8da0d1cbc35bffd830e1e19 --- /dev/null +++ b/zonos2/message/tokenizer.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List + +# Import TTS message classes so they're available in globals() for deserialization +from .tts import ( # noqa: F401 + BatchTTSTokenizerMsg, + TTSDetokenizeMsg, + TTSSamplingParams, + TTSTokenizeMsg, +) +from .utils import deserialize_type, serialize_type + + +@dataclass +class BaseTokenizerMsg: + @staticmethod + def encoder(msg: BaseTokenizerMsg) -> Dict: + return serialize_type(msg) + + @staticmethod + def decoder(json: Dict) -> BaseTokenizerMsg: + return deserialize_type(globals(), json) + + +@dataclass +class BatchTokenizerMsg(BaseTokenizerMsg): + data: List[BaseTokenizerMsg] + + +@dataclass +class AbortMsg(BaseTokenizerMsg): + uid: int diff --git a/zonos2/message/tts.py b/zonos2/message/tts.py new file mode 100644 index 0000000000000000000000000000000000000000..a9681dd6a0c43490d20177cffce660cd1ae34a46 --- /dev/null +++ b/zonos2/message/tts.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List + +import torch + +from .utils import deserialize_type, serialize_type + + +@dataclass +class TTSSamplingParams: + """Sampling parameters for TTS generation.""" + + temperature: float = 1.15 + topk: int = 106 + top_p: float = 0.0 + min_p: float = 0.18 + max_tokens: int = 1024 + ignore_eos: bool = False + repetition_window: int = 50 + repetition_penalty: float = 1.2 + repetition_codebooks: int = 8 + seed: int | None = None + + +@dataclass +class BaseTTSTokenizerMsg: + """Base class for TTS tokenizer messages.""" + + @staticmethod + def encoder(msg: BaseTTSTokenizerMsg) -> Dict: + return serialize_type(msg) + + @staticmethod + def decoder(json: Dict) -> BaseTTSTokenizerMsg: + return deserialize_type(globals(), json) + + +@dataclass +class BatchTTSTokenizerMsg(BaseTTSTokenizerMsg): + """Batch of TTS tokenizer messages.""" + + data: List[BaseTTSTokenizerMsg] + + +@dataclass +class TTSTokenizeMsg(BaseTTSTokenizerMsg): + """Request to build TTS prompt frames from text (Frontend -> Tokenizer).""" + + uid: int + text: str + sampling_params: TTSSamplingParams + # Language code used for text normalization (e.g. "en_us"). + language: str = "en_us" + # Run written->spoken text normalization before tokenization. + text_normalization: bool = True + # Optional speaker embedding for voice cloning; shape: (speaker_embedding_dim,) + speaker_embedding: torch.Tensor | None = None + # Token position in the prompt sequence where speaker embedding is injected. + speaker_token_position: int = -1 + # Whether the speaker embedding should be marked as having a clean background. + clean_speaker_background: bool = False + # Whether to condition generation on the accurate-mode marker token + # (off = expressive mode). + accurate_mode: bool = True + # Optional speaking-rate conditioning bucket. The tokenizer turns this into + # one text-column token before normal text. + speaking_rate_bucket: int | None = None + # Optional per-feature quality bucket indices (aligned with the model's + # configured quality features). Each becomes one text-column token. + quality_buckets: List[int | None] | None = None + + +@dataclass +class TTSDetokenizeMsg(BaseTTSTokenizerMsg): + """TTS output frame to vocoder (Scheduler -> Tokenizer). + + Contains audio codes for one generated frame. + """ + + uid: int + audio_codes: List[int] # [cb0, cb1, ..., cb8] for one frame + finished: bool + eos_frame: int | None = None + + +@dataclass +class BaseTTSBackendMsg: + """Base class for TTS backend messages.""" + + def encoder(self) -> Dict: + return serialize_type(self) + + @staticmethod + def decoder(json: Dict) -> BaseTTSBackendMsg: + return deserialize_type(globals(), json) + + +@dataclass +class BatchTTSBackendMsg(BaseTTSBackendMsg): + """Batch of TTS backend messages.""" + + data: List[BaseTTSBackendMsg] + + +@dataclass +class TTSUserMsg(BaseTTSBackendMsg): + """Prompt-tokenized TTS request to scheduler (Tokenizer -> Scheduler). + + Contains 2D token tensor in unpacked format. + """ + + uid: int + input_ids: torch.Tensor # 2D tensor (seq_len, frame_width) + sampling_params: TTSSamplingParams + # Optional speaker embedding for voice cloning; shape: (speaker_embedding_dim,) + speaker_embedding: torch.Tensor | None = None + # Token position in the prompt sequence where speaker embedding is injected. + speaker_token_position: int = -1 + # Whether the speaker embedding should be marked as having a clean background. + clean_speaker_background: bool = False + # Whether to condition generation on the accurate-mode marker token. + accurate_mode: bool = True + + +@dataclass +class BaseTTSFrontendMsg: + """Base class for TTS frontend messages.""" + + @staticmethod + def encoder(msg: BaseTTSFrontendMsg) -> Dict: + return serialize_type(msg) + + @staticmethod + def decoder(json: Dict) -> BaseTTSFrontendMsg: + return deserialize_type(globals(), json) + + +@dataclass +class BatchTTSFrontendMsg(BaseTTSFrontendMsg): + """Batch of TTS frontend messages.""" + + data: List[BaseTTSFrontendMsg] + + +@dataclass +class TTSAudioReply(BaseTTSFrontendMsg): + """Audio chunk reply to frontend (Tokenizer -> Frontend). + + Contains PCM audio data for streaming playback. + """ + + uid: int + audio_data: bytes # PCM audio chunk (float32, 44.1kHz) + finished: bool + sample_rate: int = 44100 diff --git a/zonos2/message/utils.py b/zonos2/message/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..41d7db62423810b72c221b8b089a77a4544219d9 --- /dev/null +++ b/zonos2/message/utils.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import Any, Dict, Type + +import numpy as np +import torch + + +def _serialize_any(value: Any) -> Any: + if isinstance(value, dict): + return {k: _serialize_any(v) for k, v in value.items()} + elif isinstance(value, (list, tuple)): + return type(value)(_serialize_any(v) for v in value) + elif isinstance(value, (int, float, str, type(None), bool, bytes)): + return value + else: + return serialize_type(value) + + +def serialize_type(self) -> Dict: + # find all member variables + serialized = {} + + if isinstance(self, torch.Tensor): + serialized["__type__"] = "Tensor" + serialized["buffer"] = self.numpy().tobytes() + serialized["dtype"] = str(self.dtype) + serialized["shape"] = list(self.shape) + return serialized + + # normal type + serialized["__type__"] = self.__class__.__name__ + for k, v in self.__dict__.items(): + serialized[k] = _serialize_any(v) + return serialized + + +def _deserialize_any(cls_map: Dict[str, Type], data: Any) -> Any: + if isinstance(data, dict): + if "__type__" in data: + return deserialize_type(cls_map, data) + else: + return {k: _deserialize_any(cls_map, v) for k, v in data.items()} + elif isinstance(data, (list, tuple)): + return type(data)(_deserialize_any(cls_map, d) for d in data) + elif isinstance(data, (int, float, str, type(None), bool, bytes)): + return data + else: + raise ValueError(f"Cannot deserialize type {type(data)}") + + +def deserialize_type(cls_map: Dict[str, Type], data: Dict) -> Any: + type_name = data["__type__"] + if type_name == "Tensor": + buffer = data["buffer"] + dtype_str = data["dtype"].replace("torch.", "") + np_dtype = getattr(np, dtype_str) + assert isinstance(buffer, bytes) + np_tensor = np.frombuffer(buffer, dtype=np_dtype) + shape = data.get("shape") + if shape is not None: + np_tensor = np_tensor.reshape(shape) + return torch.from_numpy(np_tensor.copy()) + + cls = cls_map[type_name] + kwargs = {} + for k, v in data.items(): + if k == "__type__": + continue + kwargs[k] = _deserialize_any(cls_map, v) + return cls(**kwargs) diff --git a/zonos2/models/__init__.py b/zonos2/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c6cf0b20b739775e1ac066b0d2e203cda3b189e6 --- /dev/null +++ b/zonos2/models/__init__.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from .config import ModelConfig, RotaryConfig +from .weight import load_checkpoint_weight +from .zonos2 import Zonos2ForCausalLM + + +def create_model(config: ModelConfig) -> Zonos2ForCausalLM: + return Zonos2ForCausalLM(config.model_config, config.moe_backend) + + +__all__ = [ + "Zonos2ForCausalLM", + "load_checkpoint_weight", + "create_model", + "ModelConfig", + "RotaryConfig", +] diff --git a/zonos2/models/config.py b/zonos2/models/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c559a3539b69bfef73997ab8fc5d7f9d89fda281 --- /dev/null +++ b/zonos2/models/config.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict + +from zonos2.utils.moe_topk import normalize_special_topk_layers, resolve_layer_topk + + +@dataclass(frozen=True) +class RotaryConfig: + head_dim: int + rotary_dim: int + max_position: int + base: float + scaling: Dict[str, Any] | None + + +def normalize_moe_balancing_strategy(strategy: str) -> str: + normalized = strategy.strip().lower().replace("-", "_") + aliases = { + "current": "quantile", + "quantile": "quantile", + "qbalancing": "quantile", + "old": "legacy", + "legacy": "legacy", + "aux": "legacy", + "aux_loss": "legacy", + } + try: + return aliases[normalized] + except KeyError as exc: + raise ValueError( + "Unsupported model.moe_balancing_strategy=" + f"{strategy!r}. Expected one of: current, quantile, qbalancing, old, legacy, aux_loss." + ) from exc + + +@dataclass(frozen=True) +class ModelConfig: + num_layers: int + num_qo_heads: int + num_kv_heads: int + head_dim: int + hidden_size: int + vocab_size: int + intermediate_size: int + rms_norm_eps: float + rotary_config: RotaryConfig + hidden_act: str + tie_word_embeddings: bool + num_experts: int + num_experts_per_tok: int + moe_intermediate_size: int + norm_topk_prob: bool + special_topk_layers: dict[int, int] | None = None + # Zonos2-specific fields + n_codebooks: int = 9 + codebook_size: int = 1024 + text_vocab: int | None = None + moe_n_experts: int = 1 + moe_start_from_layer: int = 0 + moe_end_from_layer: int = 0 + moe_router_dim: int = 256 # Router hidden dimension + moe_impl: str = "grouped" + moe_balancing_strategy: str = "legacy" + # TTS-specific fields + eoa_id: int = 1024 # End-of-audio token ID + audio_pad_id: int = 1025 # Audio padding token ID + loss_softcap: float = 15.0 # Logit soft capping value + speaker_enabled: bool = False + speaker_embedding_dim: int = 128 + speaker_lda_dim: int | None = None + speaker_background_token_enabled: bool = False + accurate_mode_token_enabled: bool = False + speaking_rate_num_buckets: int = 0 + speaking_rate_buckets: tuple[str, ...] = () + quality_num_buckets: int = 0 + quality_features: tuple[str, ...] = () + quality_buckets: Dict[str, tuple[str, ...]] | None = None + quality_dropout: Dict[str, float] | None = None + + def __post_init__(self) -> None: + object.__setattr__( + self, + "special_topk_layers", + normalize_special_topk_layers(self.special_topk_layers), + ) + + def get_num_experts_per_tok(self, layer_idx: int) -> int: + default_topk = self.num_experts_per_tok if self.num_experts_per_tok > 0 else 1 + return resolve_layer_topk(default_topk, self.special_topk_layers, layer_idx) + + @classmethod + def from_checkpoint_config(cls, config) -> ModelConfig: + return cls.from_zonos2_config(config) + + @classmethod + def from_zonos2_config(cls, config) -> ModelConfig: + """Create ModelConfig from Zonos2Config (training checkpoint format). + + Zonos2Config uses different field names than HuggingFace: + - n_layers -> num_layers + - dim -> hidden_size + - head_dim -> head_dim + - n_heads -> num_qo_heads + - n_kv_heads -> num_kv_heads + - ffn_dim_multiplier * dim -> intermediate_size + """ + # Calculate dimensions + head_dim = config.head_dim + dim = config.dim + n_heads = config.n_heads if config.n_heads is not None else dim // head_dim + n_kv_heads = config.n_kv_heads if config.n_kv_heads is not None else n_heads + + # Calculate intermediate size from ffn_dim_multiplier + ffn_dim = int(config.ffn_dim_multiplier * dim) + # Round to multiple_of + multiple_of = getattr(config, "multiple_of", 256) + intermediate_size = multiple_of * ((ffn_dim + multiple_of - 1) // multiple_of) + + # Calculate vocab size for audio codebooks + # Total vocab = n_codebooks * (codebook_size + 2) + text_vocab (if present) + audio_vocab = config.codebook_size + 2 # +2 for eoa_id and audio_pad_id + vocab_size = config.n_codebooks * audio_vocab + if config.text_vocab is not None: + vocab_size += config.text_vocab + 1 # +1 for text padding + + # MoE settings + moe_n_experts = getattr(config, "moe_n_experts", 1) + moe_router_topk = getattr(config, "moe_router_topk", 1) + special_topk_layers = getattr(config, "special_topk_layers", None) + moe_router_dim = getattr(config, "moe_router_dim", 256) + moe_impl = getattr(config, "moe_impl", "grouped") + moe_balancing_strategy = normalize_moe_balancing_strategy( + getattr(config, "moe_balancing_strategy", "legacy") + ) + + return cls( + num_layers=config.n_layers, + num_qo_heads=n_heads, + num_kv_heads=n_kv_heads, + head_dim=head_dim, + hidden_size=dim, + vocab_size=vocab_size, + intermediate_size=intermediate_size, + hidden_act="silu", # Zonos2 uses SiLU + rms_norm_eps=config.norm_eps, + tie_word_embeddings=False, + rotary_config=RotaryConfig( + head_dim=head_dim, + rotary_dim=head_dim, + max_position=config.max_seqlen, + base=config.rope_theta, + scaling=None, + ), + num_experts=moe_n_experts if moe_n_experts > 1 else 0, + num_experts_per_tok=moe_router_topk if moe_n_experts > 1 else 0, + special_topk_layers=special_topk_layers, + moe_intermediate_size=intermediate_size if moe_n_experts > 1 else 0, + norm_topk_prob=False, + # Zonos2 fields + n_codebooks=config.n_codebooks, + codebook_size=config.codebook_size, + text_vocab=config.text_vocab, + moe_n_experts=moe_n_experts, + moe_start_from_layer=getattr(config, "moe_start_from_layer", 0), + moe_end_from_layer=getattr(config, "moe_end_from_layer", 0), + moe_router_dim=moe_router_dim, + moe_impl=moe_impl, + moe_balancing_strategy=moe_balancing_strategy, + # TTS fields + eoa_id=config.eoa_id, + audio_pad_id=config.audio_pad_id, + loss_softcap=config.loss_softcap, + speaker_enabled=getattr(config, "speaker_enabled", False), + speaker_embedding_dim=getattr(config, "speaker_embedding_dim", 128), + speaker_lda_dim=getattr(config, "speaker_lda_dim", None), + speaker_background_token_enabled=bool( + getattr(config, "speaker_background_token_enabled", False) + ), + accurate_mode_token_enabled=bool( + getattr(config, "accurate_mode_token_enabled", False) + ), + speaking_rate_num_buckets=getattr(config, "speaking_rate_num_buckets", 0), + speaking_rate_buckets=tuple(getattr(config, "speaking_rate_buckets", None) or ()), + quality_num_buckets=int(getattr(config, "quality_num_buckets", 0) or 0), + quality_features=tuple(getattr(config, "quality_features", None) or ()), + quality_buckets={ + str(feature): tuple(str(item) for item in (buckets or ())) + for feature, buckets in (getattr(config, "quality_buckets", None) or {}).items() + } + or None, + quality_dropout=dict(getattr(config, "quality_dropout", None) or {}) or None, + ) diff --git a/zonos2/models/speaker_cloning.py b/zonos2/models/speaker_cloning.py new file mode 100644 index 0000000000000000000000000000000000000000..7e1ae0875f6a23c999cd1c2768aca8f14413801d --- /dev/null +++ b/zonos2/models/speaker_cloning.py @@ -0,0 +1,81 @@ +from functools import cache + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio +from transformers import AutoModel + +try: + from zonos.utils import DEFAULT_DEVICE +except Exception: + DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + + +class Qwen3SpeakerEmbedding(nn.Module): + """Qwen3 voice embedding extractor used by 2048D speaker-conditioned checkpoints.""" + + MODEL_NAME = "marksverdhei/Qwen3-Voice-Embedding-12Hz-1.7B" + TARGET_SAMPLE_RATE = 24_000 + N_FFT = 1024 + HOP_LENGTH = 256 + WIN_LENGTH = 1024 + N_MELS = 128 + F_MIN = 0.0 + F_MAX = 12_000.0 + + def __init__(self, device: str = DEFAULT_DEVICE): + super().__init__() + self.device = device + self.model = AutoModel.from_pretrained( + self.MODEL_NAME, + trust_remote_code=True, + ) + self.model.to(device) + self.model.eval() + + self.mel_transform = torchaudio.transforms.MelSpectrogram( + sample_rate=self.TARGET_SAMPLE_RATE, + n_fft=self.N_FFT, + win_length=self.WIN_LENGTH, + hop_length=self.HOP_LENGTH, + f_min=self.F_MIN, + f_max=self.F_MAX, + n_mels=self.N_MELS, + power=1.0, + center=False, + norm="slaney", + mel_scale="slaney", + ).to(device) + + self.requires_grad_(False).eval() + + @property + def dtype(self): + return next(self.model.parameters()).dtype + + @cache + def _get_resampler(self, orig_sample_rate: int): + return torchaudio.transforms.Resample(orig_sample_rate, self.TARGET_SAMPLE_RATE).to(self.device) + + def prepare_input(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor: + assert wav.ndim < 3 + if wav.ndim == 2: + wav = wav.mean(0, keepdim=True) + wav = wav.to(self.device, torch.float32) + if sample_rate != self.TARGET_SAMPLE_RATE: + wav = self._get_resampler(sample_rate)(wav) + return wav + + def _make_mel(self, wav: torch.Tensor) -> torch.Tensor: + # Mirror the standalone demo preprocessing: reflect-pad, magnitude mel, log clamp. + pad = (self.N_FFT - self.HOP_LENGTH) // 2 + wav = F.pad(wav.unsqueeze(1), (pad, pad), mode="reflect").squeeze(1) + mel = self.mel_transform(wav) + mel = torch.log(torch.clamp(mel, min=1e-5)) + return mel.transpose(1, 2) + + def forward(self, wav: torch.Tensor, sample_rate: int): + wav = self.prepare_input(wav, sample_rate) + mel = self._make_mel(wav) + return self.model(input_values=mel).last_hidden_state.to(torch.float32) diff --git a/zonos2/models/speaker_lda.py b/zonos2/models/speaker_lda.py new file mode 100644 index 0000000000000000000000000000000000000000..f2487a6ce0a7bd5750422510ae11cb9a4de8ca7b --- /dev/null +++ b/zonos2/models/speaker_lda.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import torch +import torch.nn.functional as F +from zonos2.layers import BaseOP + + +class SpeakerLDAProjection(BaseOP): + """Affine LDA projection applied to raw speaker embeddings. + + The weights are stored in the model checkpoint as + ``speaker_lda_projection.weight`` and ``speaker_lda_projection.bias``. + """ + + def __init__(self, input_dim: int, output_dim: int): + self.weight = torch.empty(int(output_dim), int(input_dim)) + self.bias = torch.empty(int(output_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) diff --git a/zonos2/models/weight.py b/zonos2/models/weight.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd886e4972380ff51056bb72edfaeacbf908b7f --- /dev/null +++ b/zonos2/models/weight.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict + +import torch +from zonos2.distributed import get_tp_info +from zonos2.utils import divide_up + + +def _shard_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Shard Zonos2 checkpoint tensors for tensor parallelism.""" + shard_state_dict: Dict[str, torch.Tensor] = {} + tp_info = get_tp_info() + r = tp_info.rank + n = tp_info.size + + split_dim_0 = [ + ".wq", + ".gater", + ] + split_dim_1 = [ + ".wo", + ".w_out", + ] + + for key, value in state_dict.items(): + if ".attention.temp" in key: + shard_state_dict[key] = value.chunk(n, dim=1)[r] + elif ".wkv." in key: + if value.dim() == 3: + shard_state_dict[key] = value.chunk(n, dim=1)[r] + else: + shard_state_dict[key] = value.chunk(n, dim=0)[r] + elif ".w_in." in key: + if value.dim() == 3: + shard_state_dict[key] = value.chunk(n, dim=1)[r] + else: + shard_state_dict[key] = value.chunk(n, dim=0)[r] + elif ".experts." in key: + if "gate_up_proj" in key or "w1" in key or "w3" in key: + if value.dim() == 3: + shard_state_dict[key] = value.chunk(n, dim=1)[r] + else: + shard_state_dict[key] = value.chunk(n, dim=0)[r] + elif "down_proj" in key or "w2" in key: + if value.dim() == 3: + shard_state_dict[key] = value.chunk(n, dim=2)[r] + else: + shard_state_dict[key] = value.chunk(n, dim=1)[r] + else: + shard_state_dict[key] = value + elif any(key.count(sub) for sub in split_dim_0): + shard_state_dict[key] = value.chunk(n, dim=0)[r] + elif any(key.count(sub) for sub in split_dim_1): + shard_state_dict[key] = value.chunk(n, dim=1)[r] + elif key.count("embedders") or key.count("multi_output"): + num_embeddings = value.shape[0] + num_embeddings_per_partition = divide_up(num_embeddings, n) + vocab_start_idx = r * num_embeddings_per_partition + vocab_end_idx = min((r + 1) * num_embeddings_per_partition, num_embeddings) + shard_state_dict[key] = value[vocab_start_idx:vocab_end_idx, :] + else: + shard_state_dict[key] = value + return shard_state_dict + + +def _normalize_zonos2_state_dict( + state_dict: Dict[str, torch.Tensor], +) -> Dict[str, torch.Tensor]: + """Normalize training checkpoint keys for inference.""" + from zonos2.utils import init_logger + + logger = init_logger(__name__) + + keys_to_remap = [] + keys_to_remove = [] + for key in list(state_dict.keys()): + if ".parametrizations." in key and ".original" in key: + new_key = key.replace(".parametrizations.", ".").replace(".original", "") + keys_to_remap.append((key, new_key)) + elif ".router.ent_denom" in key or ".router.normalized_entropy" in key: + keys_to_remove.append(key) + + for old_key, new_key in keys_to_remap: + logger.info("Remapping parametrized key: %s -> %s", old_key, new_key) + state_dict[new_key] = state_dict.pop(old_key) + + for key in keys_to_remove: + logger.info("Removing training-only key: %s", key) + state_dict.pop(key) + + return state_dict + + +def _single_file_checkpoint(path: str) -> Path | None: + """Return the consolidated single-file checkpoint inside a directory, if any.""" + checkpoint_dir = Path(path) + if not checkpoint_dir.is_dir(): + return None + for name in ("model.pth", "model.pt", "consolidated/consolidated.pth"): + candidate = checkpoint_dir / name + if candidate.is_file(): + return candidate + return None + + +def _load_torch_weight(path: str) -> Dict[str, torch.Tensor]: + state = torch.load(path, map_location="cpu", weights_only=False) + if "model" in state: + return state["model"] + return state + + +def load_checkpoint_weight(model_path: str, device: torch.device) -> Dict[str, torch.Tensor]: + """Load a Zonos2 TTS checkpoint (consolidated model.pth + params.json). + + Accepts a local path or a Hugging Face repo id (e.g. ``Zyphra/ZONOS2``). + """ + from zonos2.utils import init_logger, resolve_model_path + + logger = init_logger(__name__) + model_path = resolve_model_path(model_path) + single_file = _single_file_checkpoint(model_path) + if single_file is not None: + logger.info("Loading consolidated checkpoint from %s", single_file) + state_dict = _load_torch_weight(str(single_file)) + elif os.path.isfile(model_path) and model_path.endswith((".pt", ".pth")): + state_dict = _load_torch_weight(model_path) + else: + raise ValueError( + f"Unsupported checkpoint at {model_path!r}. Expected a directory " + "containing model.pth (with params.json) or a direct .pt/.pth file." + ) + + if get_tp_info().size > 1: + state_dict = _shard_state_dict(state_dict) + + state_dict = _normalize_zonos2_state_dict(state_dict) + return {k: v.to(device) for k, v in state_dict.items()} diff --git a/zonos2/models/zonos2.py b/zonos2/models/zonos2.py new file mode 100644 index 0000000000000000000000000000000000000000..33403212b9780f724f4a883ccabcb39abbc9924b --- /dev/null +++ b/zonos2/models/zonos2.py @@ -0,0 +1,925 @@ +"""Zonos2 model matching checkpoint naming from zonos2 training.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Optional, Tuple + +import torch +import torch.nn.functional as F +from zonos2.core import get_global_ctx +from zonos2.distributed import get_tp_info +from zonos2.layers import ( + AttentionLayer, + BaseOP, + ChunkedLinear, + LinearRowParallel, + OPList, + RMSNorm, + RMSNormFused, + VocabParallelEmbedding, +) +from zonos2.layers.linear import LinearReplicated +from zonos2.layers.moe.fused_moe.layer import FusedMoE +from zonos2.utils import divide_even, init_logger, nvtx_annotate + +from .config import normalize_moe_balancing_strategy +from .speaker_lda import SpeakerLDAProjection + +_model_debug_counter = 0 +_logger = init_logger(__name__) + +if TYPE_CHECKING: + from .config import ModelConfig + + +class SimpleLinear(BaseOP): + """Simple linear layer with .weight attribute for checkpoint compatibility.""" + + def __init__(self, in_features: int, out_features: int, has_bias: bool = False): + self.in_features = in_features + self.out_features = out_features + self.weight = torch.empty(out_features, in_features) + self.bias = torch.empty(out_features) if has_bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) + + +class SimpleLinearOProj(BaseOP): + """Linear output projection with all-reduce for tensor parallelism. + + Has .weight attribute for checkpoint compatibility (wo.weight). + """ + + def __init__(self, in_features: int, out_features: int, has_bias: bool = False): + from zonos2.distributed import DistributedCommunicator + + self.in_features = in_features + self.out_features = out_features + self.weight = torch.empty(out_features, in_features) + self.bias = torch.empty(out_features) if has_bias else None + self._comm = DistributedCommunicator() + self._tp_size = get_tp_info().size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = F.linear(x, self.weight, self.bias) + if self._tp_size > 1: + y = self._comm.all_reduce(y) + return y + + +class MultiOutputHead(BaseOP): + """Multi-codebook output head with .weight attribute. + + Checkpoint naming: multi_output.weight + """ + + def __init__(self, hidden_size: int, output_size: int): + self.hidden_size = hidden_size + self.output_size = output_size + self.weight = torch.empty(output_size, hidden_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight) + + +class MultiEmbedding(BaseOP): + """Multi-codebook embedding for audio + text tokens. + + Checkpoint naming: multi_embedder.embedders.{N}.weight + All embedders are in a single ModuleList. + + IMPORTANT: Uses padding_idx to zero out embeddings for padding tokens. + This matches the reference implementation behavior where: + - Audio codebooks use audio_pad_id (1025) as padding + - Text embeddings use text_vocab as padding + """ + + def __init__(self, config: ModelConfig): + self.n_codebooks = config.n_codebooks + self.text_vocab = config.text_vocab + self.audio_pad_id = config.audio_pad_id + + # Store padding indices for each embedder column + # padding_indices[i] is the padding index for column i, or None if no padding + self.padding_indices: list[int | None] = [] + + # All embedders in a single list: audio codebooks first, then text. + # This matches zonos2 checkpoint naming: multi_embedder.embedders.{0..N}.weight + embedders_list = [] + + # Audio codebook embeddings (padding_idx = audio_pad_id) + for _ in range(config.n_codebooks): + embedders_list.append( + VocabParallelEmbedding( + num_embeddings=config.codebook_size + 2, + embedding_dim=config.hidden_size, + ) + ) + self.padding_indices.append(config.audio_pad_id) + + # Optional text embedding (at index n_codebooks, padding_idx = text_vocab) + if config.text_vocab is not None: + embedders_list.append( + VocabParallelEmbedding( + num_embeddings=config.text_vocab + 1, + embedding_dim=config.hidden_size, + ) + ) + self.padding_indices.append(config.text_vocab) + + self.embedders = OPList(embedders_list) + + @nvtx_annotate("MultiEmbedding") + def forward(self, codes: torch.Tensor) -> torch.Tensor: + # codes: (batch, seq, n_codebooks + extras) or (total_tokens, n_codebooks + extras) + # Sum embeddings from all provided codebooks + # Note: codes[..., i] creates non-contiguous view, must call .contiguous() + # + # NOTE: We do NOT mask out padding tokens here. The reference implementation + # relies on the embedding table having zeros at padding positions, but the + # checkpoint may have non-zero values there. To match reference behavior, + # we just sum all embeddings without masking. + + # Get first embedding + col_codes = codes[..., 0].contiguous() + result = self.embedders.op_list[0].forward(col_codes) + + # Sum remaining columns + for i in range(1, codes.size(-1)): + col_codes = codes[..., i].contiguous() + emb = self.embedders.op_list[i].forward(col_codes) + result = result + emb + + return result + + +class Attention(BaseOP): + """Attention with QK norm and headwise Qwen-style gating. + + Checkpoint naming: + - layers.{N}.attention.wq.weight + - layers.{N}.attention.wkv.weight (3D: [2, kv_dim, hidden]) + - layers.{N}.attention.wo.weight + - layers.{N}.attention.temp + - layers.{N}.attention.gater.weight + """ + + def __init__(self, config: ModelConfig, layer_id: int): + head_dim = config.head_dim + self.num_heads = config.num_qo_heads + self.num_kv_heads = config.num_kv_heads + self.head_dim = head_dim + + # Linear projections matching checkpoint naming + tp_info = get_tp_info() + local_num_heads = divide_even(self.num_heads, tp_info.size) + local_num_kv_heads = divide_even(self.num_kv_heads, tp_info.size) + + # wq: [num_heads * head_dim, hidden_size] - uses SimpleLinear for .weight attribute + self.wq = SimpleLinear(config.hidden_size, local_num_heads * head_dim, has_bias=False) + # wkv: ChunkedLinear with 3D weight [2, kv_dim, hidden] + self.wkv = ChunkedLinear( + in_features=config.hidden_size, + out_features=local_num_kv_heads * head_dim * 2, + divisor=2, + has_bias=False, + ) + # wo: [hidden_size, num_heads * head_dim] - uses SimpleLinearOProj for all-reduce + .weight attribute + self.wo = SimpleLinearOProj(local_num_heads * head_dim, config.hidden_size, has_bias=False) + + # QK normalization with learnable temperature. + # temp: [1, local_num_heads, 1] - sharded across TP ranks. + self.temp = torch.ones(1, local_num_heads, 1, dtype=torch.bfloat16) + + # Attention backend + self.attn = AttentionLayer( + layer_id=layer_id, + head_dim=head_dim, + num_qo_heads=config.num_qo_heads, + num_kv_heads=config.num_kv_heads, + rotary_config=config.rotary_config, + q_norm=None, # We handle norm ourselves for temperature scaling + k_norm=None, + ) + + # Gater output is sharded across TP ranks (local_num_heads per rank). + self.gater = SimpleLinear(config.hidden_size, local_num_heads, has_bias=False) + + @nvtx_annotate("Attention") + def forward(self, x: torch.Tensor) -> torch.Tensor: + global _model_debug_counter + ctx = get_global_ctx() + is_capturing = torch.cuda.is_current_stream_capturing() + debug_attn = ( + _logger.isEnabledFor(logging.DEBUG) and ctx.batch.is_decode and not is_capturing and + self.attn.layer_id == 0 and _model_debug_counter <= 10 + ) + + gate = torch.sigmoid(self.gater.forward(x)) + + # Q projection + q = self.wq.forward(x) + + # KV projection (from ChunkedLinear) + kv = self.wkv.forward(x) + + # Split KV into K and V + tp_info = get_tp_info() + local_num_kv_heads = divide_even(self.num_kv_heads, tp_info.size) + kv_dim = local_num_kv_heads * self.head_dim + + k, v = kv.split([kv_dim, kv_dim], dim=-1) + # v needs to be contiguous because it's passed directly to the KV cache kernel + # (k becomes contiguous through flatten/view operations in the QK norm path) + v = v.contiguous() + + if debug_attn: + step = _model_debug_counter + _logger.debug("Attn L0 Step %d Q proj: norm=%.4f", step, q.norm().item()) + _logger.debug("Attn L0 Step %d K proj: norm=%.4f", step, k.norm().item()) + _logger.debug("Attn L0 Step %d V proj: norm=%.4f", step, v.norm().item()) + + metadata = ctx.batch.attn_metadata + local_num_heads = divide_even(self.num_heads, tp_info.size) + qo_attn_dim = local_num_heads * self.head_dim + + q = q.view(-1, local_num_heads, self.head_dim) + k = k.view(-1, local_num_kv_heads, self.head_dim) + + q = F.rms_norm(q, (self.head_dim,), eps=1e-6) * self.temp.abs().to(q.dtype).to(q.device) + k = F.rms_norm(k, (self.head_dim,), eps=1e-6) + + if debug_attn: + step = _model_debug_counter + _logger.debug("Attn L0 Step %d Q after norm+temp: norm=%.4f", step, q.norm().item()) + _logger.debug("Attn L0 Step %d K after norm: norm=%.4f", step, k.norm().item()) + + # Zonos2 uses interleaved RoPE format (is_neox=False). + if self.attn.rotary: + q, k = self.attn.rotary.forward(metadata.positions, q.flatten(-2), k.flatten(-2), is_neox=False) + + if debug_attn: + step = _model_debug_counter + _logger.debug("Attn L0 Step %d Q after RoPE: norm=%.4f", step, q.norm().item()) + _logger.debug("Attn L0 Step %d K after RoPE: norm=%.4f", step, k.norm().item()) + + # q needs to be 3D for flash attention: (total_tokens, num_heads, head_dim) + # k and v stay 2D for the KV cache store kernel: (total_tokens, kv_dim) + q = q.view(-1, local_num_heads, self.head_dim) + + o = ctx.attn_backend.forward(q, k, v, self.attn.layer_id, ctx.batch) + o = o.view(-1, qo_attn_dim) + + if debug_attn: + _logger.debug("Attn L0 Step %d Attn output: norm=%.4f", _model_debug_counter, o.norm().item()) + + o = o.view(-1, local_num_heads, self.head_dim) + o = o * gate.unsqueeze(-1) + o = o.view(-1, local_num_heads * self.head_dim) + + if debug_attn: + _logger.debug("Attn L0 Step %d After gating: norm=%.4f", _model_debug_counter, o.norm().item()) + + # Output projection + result = self.wo.forward(o) + + if debug_attn: + _logger.debug("Attn L0 Step %d After wo proj: norm=%.4f", _model_debug_counter, result.norm().item()) + + return result + + +class FeedForward(BaseOP): + """Dense feedforward layer. + + Checkpoint naming: + - layers.{N}.feed_forward.w_in.weight (3D: [2, intermediate, hidden]) + - layers.{N}.feed_forward.w_out.weight + """ + + def __init__(self, config: ModelConfig): + tp_info = get_tp_info() + local_intermediate = divide_even(config.intermediate_size, tp_info.size) + + # w_in: ChunkedLinear [2, intermediate, hidden] for gate and up + self.w_in = ChunkedLinear( + in_features=config.hidden_size, + out_features=local_intermediate * 2, + divisor=2, + has_bias=False, + ) + # w_out: [hidden, intermediate] + self.w_out = LinearRowParallel( + input_size=config.intermediate_size, + output_size=config.hidden_size, + has_bias=False, + ) + self._inter_size = local_intermediate + + @nvtx_annotate("FeedForward") + def forward(self, x: torch.Tensor) -> torch.Tensor: + # w_in produces [h, gate] where h is "up" and gate gets SiLU + # Reference does: h * F.silu(gate) = up * SiLU(gate) + h_gate = self.w_in.forward(x) + inter_size = self._inter_size + h = h_gate[..., :inter_size] # first half = h (up projection) + gate = h_gate[..., inter_size:] # second half = gate + # Match reference: h * F.silu(gate) + y = h * F.silu(gate) + return self.w_out.forward(y) + + +class FusedGroupedExperts(BaseOP): + """Fused MoE experts with weight fusion during loading. + + Checkpoint naming: + - layers.{N}.feed_forward.experts.w1.weight (or per-expert: w1) + - layers.{N}.feed_forward.experts.w2.weight + - layers.{N}.feed_forward.experts.w3.weight + - layers.{N}.feed_forward.experts.w13 (SonicMoE, interleaved gate/up) + + Inference format: + - gate_up_proj: [num_experts, 2 * intermediate, hidden] = concat(w1, w3) + - down_proj: [num_experts, hidden, intermediate] = w2 + """ + + def __init__(self, config: ModelConfig, layer_id: int): + tp_info = get_tp_info() + # Use moe_n_experts for zonos2 (not num_experts which is for HF models) + self.num_experts = config.moe_n_experts + self.hidden_size = config.hidden_size + # Use intermediate_size for MoE (moe_intermediate_size may be 0) + self.intermediate_size = config.moe_intermediate_size if config.moe_intermediate_size > 0 else config.intermediate_size + self.top_k = config.get_num_experts_per_tok(layer_id) + + local_intermediate = divide_even(self.intermediate_size, tp_info.size) + + # Fused weights: gate_up_proj = [w1, w3], down_proj = w2 + self.gate_up_proj = torch.empty( + self.num_experts, 2 * local_intermediate, self.hidden_size + ) + self.down_proj = torch.empty( + self.num_experts, self.hidden_size, local_intermediate + ) + + self._fused_moe = FusedMoE( + num_experts=self.num_experts, + top_k=self.top_k, + hidden_size=self.hidden_size, + intermediate_size=self.intermediate_size, + renormalize=config.norm_topk_prob, + prefix="", + ) + # Override weights with ours + self._fused_moe.gate_up_proj = self.gate_up_proj + self._fused_moe.down_proj = self.down_proj + + def load_state_dict( + self, + state_dict, + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + """Handle loading from both fused and unfused weight formats. + + If checkpoint has w1.weight and w3.weight (unfused), fuse them into gate_up_proj. + If checkpoint has SonicMoE w13, de-interleave it into gate_up_proj. + If checkpoint has gate_up_proj, load directly. + """ + from zonos2.layers.base import _concat_prefix + + # Keys we expect in fused format + gate_up_key = _concat_prefix(prefix, "gate_up_proj") + down_proj_key = _concat_prefix(prefix, "down_proj") + + # Keys from zonos2 checkpoint (unfused format) + w1_key = _concat_prefix(prefix, "w1.weight") + w2_key = _concat_prefix(prefix, "w2.weight") + w3_key = _concat_prefix(prefix, "w3.weight") + sonic_w13_key = _concat_prefix(prefix, "w13") + sonic_w2_key = _concat_prefix(prefix, "w2") + + def _convert_sonic_w13_to_gate_up(w13: torch.Tensor) -> torch.Tensor: + assert w13.dim() == 3, f"Expected SonicMoE w13 to be rank-3, got {w13.shape}" + assert w13.shape[1] % 2 == 0, ( + f"Expected even fused width in SonicMoE w13, got {w13.shape}" + ) + gate = w13[:, 0::2, :] + up = w13[:, 1::2, :] + return torch.cat([gate, up], dim=1) + + # Check if we have unfused weights that need fusion + if gate_up_key not in state_dict and sonic_w13_key in state_dict: + state_dict[gate_up_key] = _convert_sonic_w13_to_gate_up( + state_dict.pop(sonic_w13_key) + ) + elif gate_up_key not in state_dict and w1_key in state_dict and w3_key in state_dict: + w1 = state_dict.pop(w1_key) + w3 = state_dict.pop(w3_key) + # Fuse: gate_up_proj = [w1, w3] along dim 1 + # w1 = gate projection (SiLU applied), w3 = up projection + state_dict[gate_up_key] = torch.cat([w1, w3], dim=1) + + # Handle w2 -> down_proj mapping + if down_proj_key not in state_dict and sonic_w2_key in state_dict: + state_dict[down_proj_key] = state_dict.pop(sonic_w2_key) + elif w2_key in state_dict and down_proj_key not in state_dict: + state_dict[down_proj_key] = state_dict.pop(w2_key) + + # Now load using standard mechanism + super().load_state_dict(state_dict, prefix=prefix, _internal=_internal) + + # Update _fused_moe's references to point to the newly loaded tensors + # (setattr in super().load_state_dict replaces self.gate_up_proj/down_proj, + # but _fused_moe still points to the old meta tensors) + self._fused_moe.gate_up_proj = self.gate_up_proj + self._fused_moe.down_proj = self.down_proj + + def forward( + self, + x: torch.Tensor, + router_logits: torch.Tensor | None = None, + *, + topk_weights: torch.Tensor | None = None, + topk_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + return self._fused_moe.forward( + x, + router_logits=router_logits, + topk_weights=topk_weights, + topk_ids=topk_ids, + ) + + +class RouterMLP(BaseOP): + """Router MLP layer matching zonos2's nn.Sequential structure. + + Checkpoint naming: + - router_mlp.0.weight, router_mlp.0.bias (Linear) + - router_mlp.2.weight, router_mlp.2.bias (Linear, after GELU at index 1) + - router_mlp.4.weight (Linear, after GELU at index 3) + + The GELU activations (indices 1, 3) are stateless and don't have weights. + """ + + def __init__(self, router_dim: int, num_experts: int): + # Match zonos2's nn.Sequential indices exactly + # 0: Linear(router_dim, router_dim, bias=True) + # 1: GELU() - no weights + # 2: Linear(router_dim, router_dim, bias=True) + # 3: GELU() - no weights + # 4: Linear(router_dim, num_experts, bias=False) + + # Using SimpleLinear with bias for layers 0, 2 and without bias for layer 4 + self._0 = _RouterLinear(router_dim, router_dim, has_bias=True) + self._2 = _RouterLinear(router_dim, router_dim, has_bias=True) + self._4 = _RouterLinear(router_dim, num_experts, has_bias=False) + + def state_dict(self, *, prefix: str = "", result=None): + """Custom state_dict to match nn.Sequential numeric naming.""" + from zonos2.layers.base import _concat_prefix + + result = result if result is not None else {} + self._0.state_dict(prefix=_concat_prefix(prefix, "0"), result=result) + self._2.state_dict(prefix=_concat_prefix(prefix, "2"), result=result) + self._4.state_dict(prefix=_concat_prefix(prefix, "4"), result=result) + return result + + def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False): + """Custom load_state_dict to match nn.Sequential numeric naming.""" + from zonos2.layers.base import _concat_prefix + + self._0.load_state_dict(state_dict, prefix=_concat_prefix(prefix, "0"), _internal=True) + self._2.load_state_dict(state_dict, prefix=_concat_prefix(prefix, "2"), _internal=True) + self._4.load_state_dict(state_dict, prefix=_concat_prefix(prefix, "4"), _internal=True) + + if not _internal and state_dict: + keys = list(state_dict.keys()) + raise RuntimeError( + f"Unexpected keys in state_dict: {len(keys)} keys (first 10: {keys[:10]})" + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = F.gelu(self._0.forward(x)) + x = F.gelu(self._2.forward(x)) + x = self._4.forward(x) + return x + + +class _RouterLinear(BaseOP): + """Simple linear layer for router MLP.""" + + def __init__(self, in_features: int, out_features: int, has_bias: bool = True): + self.weight = torch.empty(out_features, in_features) + self.bias = torch.empty(out_features) if has_bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) + + +class Router(BaseOP): + """MoE Router matching zonos2's EDA Router structure. + + Checkpoint naming: + - router.down_proj.weight, router.down_proj.bias + - router.router_mlp.{0,2,4}.weight, router.router_mlp.{0,2}.bias + - router.rmsnorm_eda.weight + - router.router_states_scale (optional, for EDA layers) + - router.balancing_biases + """ + + def __init__(self, config: ModelConfig, layer_id: int): + self.hidden_size = config.hidden_size + self.router_dim = config.moe_router_dim + self.num_experts = config.moe_n_experts + self.top_k = config.get_num_experts_per_tok(layer_id) + self.moe_balancing_strategy = normalize_moe_balancing_strategy( + config.moe_balancing_strategy + ) + self.use_legacy_balancing = self.moe_balancing_strategy == "legacy" + self._layer_id = layer_id + + # down_proj: Linear(hidden_size, router_dim, bias=True) + self.down_proj = _RouterLinear(self.hidden_size, self.router_dim, has_bias=True) + + # router_mlp: Sequential with GELU activations + self.router_mlp = RouterMLP(self.router_dim, self.num_experts) + + # RMSNorm for EDA + self.rmsnorm_eda = RMSNorm(size=self.router_dim, eps=config.rms_norm_eps) + + # EDA: use_eda is True for all layers except the first MoE layer + self.use_eda = layer_id != config.moe_start_from_layer + if self.use_eda: + self.router_states_scale = torch.ones(self.router_dim) + + # Balancing biases + self.balancing_biases = torch.zeros(self.num_experts, dtype=torch.float32) + + def _select_balanced_topk( + self, expert_prob: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + with torch.no_grad(): + bias = self.balancing_biases.detach().float() + routing_scores = ( + expert_prob.float() + bias + if self.use_legacy_balancing + else expert_prob.float() - bias + ) + _, expert_choice = torch.topk(routing_scores, self.top_k, dim=-1) + + route_prob = torch.gather(expert_prob, dim=-1, index=expert_choice) + return route_prob.reshape(-1, self.top_k), expert_choice.reshape(-1, self.top_k).to( + dtype=torch.int32 + ) + + def load_state_dict( + self, + state_dict, + *, + prefix: str = "", + _internal: bool = False, + ) -> None: + from zonos2.layers.base import _concat_prefix + + for name in ("router_states_scale", "balancing_biases"): + key = _concat_prefix(prefix, name) + if key in state_dict and hasattr(self, name): + current = getattr(self, name) + item = state_dict[key] + if isinstance(current, torch.Tensor) and current.dtype != item.dtype: + setattr(self, name, current.to(dtype=item.dtype)) + + super().load_state_dict(state_dict, prefix=prefix, _internal=_internal) + + def forward(self, hidden_states: torch.Tensor, router_states: Optional[torch.Tensor] = None): + """Compute router logits. + + Args: + hidden_states: [batch, hidden_size] + router_states: [batch, router_dim] from previous layer (for EDA) + + Returns: + route_prob_flat: [tokens, top_k] routing weights from pre-bias softmax + expert_choice_flat: [tokens, top_k] expert indices after bias-aware top-k + hidden_states_next: [batch, router_dim] for next layer's EDA + """ + # Down-project to router dimension + hidden_states = self.down_proj.forward(hidden_states) + + # EDA: blend with previous router states + if self.use_eda and router_states is not None: + hidden_states = hidden_states + router_states * self.router_states_scale + + # Save for next layer's EDA + hidden_states_next = hidden_states.clone() + + # Normalize + hidden_states = self.rmsnorm_eda.forward(hidden_states) + + expert_prob = torch.softmax(self.router_mlp.forward(hidden_states).float(), dim=-1) + route_prob_flat, expert_choice_flat = self._select_balanced_topk(expert_prob) + + return route_prob_flat, expert_choice_flat, hidden_states_next + + +class MoEFeedForward(BaseOP): + """MoE feedforward layer. + + Checkpoint naming: + - layers.{N}.feed_forward.router.* + - layers.{N}.feed_forward.experts.* + """ + + def __init__(self, config: ModelConfig, layer_id: int): + # Router with EDA support + self.router = Router(config, layer_id) + + # Experts with weight fusion + self.experts = FusedGroupedExperts(config, layer_id) + + @nvtx_annotate("MoEFeedForward") + def forward(self, x: torch.Tensor, router_states: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward pass for MoE layer. + + Args: + x: [num_tokens, hidden_dim] + router_states: [num_tokens, router_dim] from previous MoE layer (for EDA) + + Returns: + output: [num_tokens, hidden_dim] + router_states_next: [num_tokens, router_dim] for next MoE layer + """ + num_tokens, hidden_dim = x.shape + x_input = x.view(-1, hidden_dim) + + # Compute router weights/ids and EDA states for next layer. + topk_weights, topk_ids, router_states_next = self.router.forward( + x_input, router_states + ) + + # Expert computation + output = self.experts.forward( + x_input, + topk_weights=topk_weights, + topk_ids=topk_ids, + ) + + return output.view(num_tokens, hidden_dim), router_states_next + + +class TransformerBlock(BaseOP): + """Transformer decoder layer. + + Checkpoint naming: + - layers.{N}.attention.* + - layers.{N}.attention_norm.weight + - layers.{N}.ffn_norm.weight + - layers.{N}.feed_forward.* + """ + + def __init__(self, config: ModelConfig, layer_id: int): + self.attention = Attention(config, layer_id) + + # Norms with checkpoint naming + self.attention_norm = RMSNormFused( + size=config.hidden_size, + eps=config.rms_norm_eps, + ) + self.ffn_norm = RMSNormFused( + size=config.hidden_size, + eps=config.rms_norm_eps, + ) + + # Feed forward: MoE or dense based on layer position + self.is_moe = self._is_moe_layer(config, layer_id) + if self.is_moe: + self.feed_forward = MoEFeedForward(config, layer_id) + else: + self.feed_forward = FeedForward(config) + + self._layer_id = layer_id + + def _is_moe_layer(self, config: ModelConfig, layer_id: int) -> bool: + """Check if this layer should be MoE based on config.""" + if config.moe_n_experts <= 1: + return False + if layer_id < config.moe_start_from_layer: + return False + if (config.num_layers - layer_id) <= config.moe_end_from_layer: + return False + return True + + @nvtx_annotate("Layer_{}", layer_id_field="_layer_id") + def forward( + self, x: torch.Tensor, residual: torch.Tensor | None = None, router_states: torch.Tensor | None = None + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + batch = get_global_ctx().batch + is_capturing = torch.cuda.is_current_stream_capturing() + debug_this_layer = ( + _logger.isEnabledFor(logging.DEBUG) and batch.is_decode and not is_capturing and + self._layer_id == 0 and _model_debug_counter <= 10 + ) + + if debug_this_layer: + _logger.debug("Layer 0 Step %d Input x: norm=%.4f, mean=%.6f", _model_debug_counter, x.norm().item(), x.mean().item()) + + x, residual = self.attention_norm.forward(x, residual) + + if debug_this_layer: + _logger.debug("Layer 0 Step %d After attn_norm: x_norm=%.4f, res_norm=%.4f", _model_debug_counter, x.norm().item(), residual.norm().item()) + + x = self.attention.forward(x) + + if debug_this_layer: + _logger.debug("Layer 0 Step %d After attention: x_norm=%.4f, mean=%.6f", _model_debug_counter, x.norm().item(), x.mean().item()) + x_flat = x.view(-1)[:10] + _logger.debug("Layer 0 Step %d Attn out[:10]: %s", _model_debug_counter, [f'{v:.4f}' for v in x_flat.tolist()]) + + x, residual = self.ffn_norm.forward(x, residual) + + if debug_this_layer: + _logger.debug("Layer 0 Step %d After ffn_norm: x_norm=%.4f, res_norm=%.4f", _model_debug_counter, x.norm().item(), residual.norm().item()) + + # MoE layers return router_states for EDA + if self.is_moe: + x, router_states = self.feed_forward.forward(x, router_states) + else: + x = self.feed_forward.forward(x) + router_states = None + + return x, residual, router_states + + +def softcap(x: torch.Tensor, cap: float) -> torch.Tensor: + """Apply soft capping to logits.""" + return cap * torch.tanh(x / cap) + + +class Zonos2ForCausalLM(BaseOP): + """Zonos2 causal LM. + + Checkpoint naming (no "model." prefix): + - multi_embedder.* + - layers.{N}.* + - out_norm.weight + - multi_output.weight + """ + + def __init__(self, config: ModelConfig, moe_backend: str = "fused_moe"): + self.config = config + self.n_codebooks = config.n_codebooks + self.audio_vocab = config.codebook_size + 2 + + # All components directly on this class for correct state_dict keys + self.multi_embedder = MultiEmbedding(config) + + # Training uses elementwise_affine=False (no learnable params). + self.emb_norm = RMSNormFused( + size=config.hidden_size, + eps=config.rms_norm_eps, + elementwise_affine=False, + ) + + # Optional speaker embedding projections (voice cloning capable checkpoints). + # The LDA projection reduces raw speaker embeddings before the main + # speaker projection; its weights are merged into the checkpoint. + self.speaker_lda_projection = ( + SpeakerLDAProjection( + input_dim=config.speaker_embedding_dim, + output_dim=int(config.speaker_lda_dim), + ) + if config.speaker_enabled and config.speaker_lda_dim + else None + ) + speaker_projection_input_dim = ( + int(config.speaker_lda_dim) + if self.speaker_lda_projection is not None + else config.speaker_embedding_dim + ) + self.speaker_projection = ( + LinearReplicated( + input_size=speaker_projection_input_dim, + output_size=config.hidden_size, + has_bias=True, + ) + if config.speaker_enabled + else None + ) + + self.layers = OPList( + [TransformerBlock(config, layer_id) for layer_id in range(config.num_layers)] + ) + + self.out_norm = RMSNormFused( + size=config.hidden_size, + eps=config.rms_norm_eps, + ) + + # Multi-output head: single linear [audio_vocab * n_codebooks, hidden] + # Uses MultiOutputHead wrapper for checkpoint compatibility (multi_output.weight) + self.multi_output = MultiOutputHead(config.hidden_size, self.audio_vocab * self.n_codebooks) + + super().__init__() + + def _forward_model(self, input_ids: torch.Tensor) -> torch.Tensor: + """Run model forward pass to get hidden states.""" + global _model_debug_counter + batch = get_global_ctx().batch + is_decode = batch.is_decode + + is_capturing = torch.cuda.is_current_stream_capturing() + should_debug = is_decode and _logger.isEnabledFor(logging.DEBUG) and not is_capturing + + if should_debug: + _model_debug_counter += 1 + step = _model_debug_counter + if step <= 10: + _logger.debug("MODEL Step %d input_ids shape: %s", step, input_ids.shape) + _logger.debug(" input_ids[0]: %s", input_ids[0].tolist()) + + x = self.multi_embedder.forward(input_ids) + + if should_debug and _model_debug_counter <= 10: + _logger.debug(" After embedding: norm=%.4f, mean=%.6f", x.norm().item(), x.mean().item()) + + # Inject projected speaker embeddings at selected token positions. + if self.speaker_projection is not None: + speaker_emb_values = getattr(batch, "speaker_emb_values", None) + speaker_token_positions = getattr(batch, "speaker_token_positions", None) + if ( + speaker_emb_values is not None + and speaker_token_positions is not None + and speaker_emb_values.numel() > 0 + and speaker_token_positions.numel() > 0 + ): + if self.speaker_lda_projection is not None: + speaker_emb_values = self.speaker_lda_projection.forward( + speaker_emb_values.to(dtype=self.speaker_lda_projection.weight.dtype) + ) + projected_speaker = self.speaker_projection.forward( + speaker_emb_values.to(dtype=self.speaker_projection.weight.dtype) + ) + x = x.index_copy(0, speaker_token_positions, projected_speaker.to(dtype=x.dtype)) + + x, _ = self.emb_norm.forward(x, None) + + if should_debug and _model_debug_counter <= 10: + _logger.debug(" After emb_norm: norm=%.4f, mean=%.6f", x.norm().item(), x.mean().item()) + + residual: torch.Tensor | None = None + router_states: torch.Tensor | None = None + for layer_id, layer in enumerate(self.layers.op_list): + x, residual, router_states = layer.forward(x, residual, router_states) + + if should_debug and _model_debug_counter <= 10: + if layer_id in [0, 1, 5, 10, 15, len(self.layers.op_list) - 1]: + res_norm = residual.norm().item() if residual is not None else 0 + _logger.debug(" After layer %d: x_norm=%.4f, x_mean=%.6f, res_norm=%.4f", layer_id, x.norm().item(), x.mean().item(), res_norm) + + h = self.out_norm.forward(x, residual)[0] + + if should_debug and _model_debug_counter <= 10: + _logger.debug(" After out_norm: norm=%.4f, mean=%.6f", h.norm().item(), h.mean().item()) + + return h + + def forward(self) -> torch.Tensor: + batch = get_global_ctx().batch + hidden = self._forward_model(batch.input_ids) + + is_capturing = torch.cuda.is_current_stream_capturing() + if (batch.is_decode and _logger.isEnabledFor(logging.DEBUG) + and not is_capturing and _model_debug_counter <= 10): + _logger.debug(" Hidden for logits: norm=%.4f, mean=%.6f", hidden.norm().item(), hidden.mean().item()) + h_flat = hidden.view(-1)[:10] + _logger.debug(" Hidden[:10]: %s", [f'{v:.4f}' for v in h_flat.tolist()]) + + return self.compute_logits(hidden) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Compute multi-codebook logits from hidden states. + + Args: + hidden_states: [batch, seq, hidden] or [total_tokens, hidden] + + Returns: + logits: [..., n_codebooks, audio_vocab] + """ + # Linear projection: [..., audio_vocab * n_codebooks] + logits = self.multi_output.forward(hidden_states) + # Reshape to [..., n_codebooks, audio_vocab] + *batch_dims, _ = logits.shape + logits = logits.view(*batch_dims, self.n_codebooks, self.audio_vocab) + + # Apply soft capping if configured + if self.config.loss_softcap > 0: + logits = softcap(logits, self.config.loss_softcap) + + return logits + + +__all__ = ["Zonos2ForCausalLM"] diff --git a/zonos2/scheduler/__init__.py b/zonos2/scheduler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..846fdee0da0ab7772417fc42292f5412eaa8982d --- /dev/null +++ b/zonos2/scheduler/__init__.py @@ -0,0 +1,4 @@ +from .config import SchedulerConfig +from .scheduler import TTSScheduler + +__all__ = ["SchedulerConfig", "TTSScheduler"] diff --git a/zonos2/scheduler/cache.py b/zonos2/scheduler/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8cf8725f1a268d06cffb3fda98b2583f9e345b --- /dev/null +++ b/zonos2/scheduler/cache.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from zonos2.kvcache import BaseCacheHandle, create_cache_manager + +if TYPE_CHECKING: + from .utils import PendingReq + + +class CacheManager: + def __init__(self, device: torch.device, num_pages: int, type: str): + # TODO: support page_size > 1 + self._free_slots = torch.arange(num_pages, dtype=torch.int32, device=device) + self.device = device + self.manager = create_cache_manager(device=device, type=type) + self.num_pages = num_pages + + def _free(self, indices: torch.Tensor) -> None: + if len(indices) > 0: + self._free_slots = torch.cat([self._free_slots, indices]) + + def match_req(self, req: PendingReq): + input_len = req.input_len + assert input_len > 0, "Input length must be greater than 0." + return self.manager.match_prefix(req.input_ids[: input_len - 1]) + + def allocate_new_handle(self) -> BaseCacheHandle: + """Allocate a new empty cache handle (for TTS which doesn't use prefix caching).""" + # Use match_prefix with empty tensor to get an empty handle + handle, _ = self.manager.match_prefix(torch.empty(0, dtype=torch.int32, device=self.device)) + return handle + + def free_handle(self, handle: BaseCacheHandle) -> None: + """Free a cache handle (for TTS which doesn't use prefix caching).""" + # For TTS, we just unlock the handle - no prefix insertion needed + self.unlock(handle) + + def free_slots(self, indices: torch.Tensor) -> None: + """Free cache slots directly (for TTS which doesn't use prefix caching).""" + self._free(indices) + + @property + def available_size(self) -> int: + return self.manager.size_info.evictable_size + len(self._free_slots) + + def lock(self, handle: BaseCacheHandle) -> None: + self.manager.lock_handle(handle, unlock=False) + + def unlock(self, handle: BaseCacheHandle) -> None: + self.manager.lock_handle(handle, unlock=True) + + def allocate(self, needed_len: int) -> torch.Tensor: + if needed_len <= (free_len := len(self._free_slots)): + allocated = self._free_slots[:needed_len] + self._free_slots = self._free_slots[needed_len:] + return allocated + + # NOTE: len(evicted) + free_len >= needed_len + evicted = self.manager.evict(needed_len - free_len) + merged = torch.cat([self._free_slots, evicted]) + assert len(merged) >= needed_len, "Eviction did not free enough space." + + allocated = merged[:needed_len] + self._free_slots = merged[needed_len:] + return allocated + + def free_and_cache_finished_req( + self, + old_handle: BaseCacheHandle, + input_ids: torch.Tensor, + indices: torch.Tensor, + ) -> None: + in_cache_len = self.manager.insert_prefix(input_ids, indices) + self._free(indices[old_handle.cached_len : in_cache_len]) + self.unlock(old_handle) + + def check_integrity(self) -> None: + self.manager.check_integrity() + if len(self._free_slots) + self.manager.size_info.total_size != self.num_pages: + raise RuntimeError( + "CacheManager integrity check failed:" + f" free_slots({len(self._free_slots)}) +" + f" total_size({self.manager.size_info.total_size}) != num_pages({self.num_pages})" + ) diff --git a/zonos2/scheduler/config.py b/zonos2/scheduler/config.py new file mode 100644 index 0000000000000000000000000000000000000000..7a08bf683d34334527a053b36ae00c4736a23157 --- /dev/null +++ b/zonos2/scheduler/config.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from zonos2.engine import EngineConfig + + +def _get_pid_suffix() -> str: + import os + + return f".pid={os.getpid()}" + + +@dataclass(frozen=True) +class SchedulerConfig(EngineConfig): + max_extend_tokens: int = 8192 + cache_type: str = "radix" + offline_mode: bool = False + + # networking config + _unique_suffix: str = field(default_factory=_get_pid_suffix) + + @property + def zmq_backend_addr(self) -> str: + return "ipc:///tmp/zonos2_0" + self._unique_suffix + + @property + def zmq_detokenizer_addr(self) -> str: + return "ipc:///tmp/zonos2_1" + self._unique_suffix + + @property + def zmq_scheduler_broadcast_addr(self) -> str: + return "ipc:///tmp/zonos2_2" + self._unique_suffix + + @property + def max_forward_len(self) -> int: + return self.max_extend_tokens + + @property + def backend_create_detokenizer_link(self) -> bool: + return True diff --git a/zonos2/scheduler/decode.py b/zonos2/scheduler/decode.py new file mode 100644 index 0000000000000000000000000000000000000000..bb37d410c94985d7353ba84952d51717cf100581 --- /dev/null +++ b/zonos2/scheduler/decode.py @@ -0,0 +1,39 @@ +"""TTS decode manager for handling running TTS requests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Iterable, Set + +from zonos2.core import TTSBatch, TTSReq + + +@dataclass +class TTSDecodeManager: + """Manager for running TTS decode requests.""" + + running_reqs: Set[TTSReq] = field(default_factory=set) + + def add_reqs(self, reqs: Iterable[TTSReq]) -> None: + """Add requests that can still decode.""" + self.running_reqs.update(req for req in reqs if req.can_decode()) + + def remove_req(self, req: TTSReq) -> None: + """Remove a finished request.""" + self.running_reqs.discard(req) + + @property + def inflight_tokens(self) -> int: + """Total remaining tokens across all running requests.""" + return sum(req.remain_len for req in self.running_reqs) + + def schedule_next_batch(self) -> TTSBatch | None: + """Schedule the next decode batch.""" + if not self.runnable: + return None + return TTSBatch(reqs=list(self.running_reqs), phase="decode") + + @property + def runnable(self) -> bool: + """Whether there are requests that can be scheduled.""" + return bool(self.running_reqs) diff --git a/zonos2/scheduler/io.py b/zonos2/scheduler/io.py new file mode 100644 index 0000000000000000000000000000000000000000..93921d2fa2b3afb91f13120470d5d27b324515e0 --- /dev/null +++ b/zonos2/scheduler/io.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, List + +import torch +from zonos2.message import BaseBackendMsg, BaseTokenizerMsg, BatchTokenizerMsg +from zonos2.utils import ZmqPubQueue, ZmqPullQueue, ZmqPushQueue, ZmqSubQueue, init_logger + +if TYPE_CHECKING: + from .config import SchedulerConfig + +logger = init_logger(__name__) + + +class SchedulerIOMixin: + """ + Mixin class for Scheduler I/O operations. + + This class handles the communication between the scheduler and the tokenizer. + + Public Utilities: + receive_msg: Function to receive messages from the tokenizer. + send_result: Function to send results back to the tokenizer. + sync_all_ranks: Function to synchronize all ranks on CPU side. + """ + + def __init__(self, config: SchedulerConfig, tp_cpu_group: torch.distributed.ProcessGroup): + tp_info = config.tp_info + self.tp_cpu_group: Final = tp_cpu_group + if config.offline_mode: + self.receive_msg = self.offline_receive_msg + self.send_result = self.offline_send_result + return # early exit + + if tp_info.is_primary(): + self._recv_from_tokenizer: Final = ZmqPullQueue( + config.zmq_backend_addr, + create=True, + decoder=BaseBackendMsg.decoder, + ) + self._send_into_tokenizer: Final = ZmqPushQueue( + config.zmq_detokenizer_addr, + create=config.backend_create_detokenizer_link, + encoder=BaseTokenizerMsg.encoder, + ) + + recv = self._recv_msg_single_rank + send = self._reply_tokenizer_rank0 + if tp_info.size > 1: + if tp_info.is_primary(): + recv = self._recv_msg_multi_rank0 + self._send_into_ranks: Final = ZmqPubQueue( + config.zmq_scheduler_broadcast_addr, create=True, encoder=BaseBackendMsg.encoder + ) + else: + recv = self._recv_msg_multi_rank1 + send = self._reply_tokenizer_rank1 + self._recv_from_rank0: Final = ZmqSubQueue( + config.zmq_scheduler_broadcast_addr, + create=False, + decoder=BaseBackendMsg.decoder, + ) + + self.receive_msg = recv + self.send_result = send + + def run_when_idle(self): + raise NotImplementedError("should be implemented") + + def offline_receive_msg(self, blocking: bool = False) -> List[BaseBackendMsg]: + raise NotImplementedError("should be implemented") + + def offline_send_result(self, reply: BatchTokenizerMsg) -> None: + raise NotImplementedError("should be implemented") + + def sync_all_ranks(self) -> None: + self.tp_cpu_group.barrier().wait() + + def _recv_msg_single_rank(self, blocking: bool = False) -> List[BaseBackendMsg]: + pending_msgs: List[BaseBackendMsg] = [] + if blocking: + self.run_when_idle() + pending_msgs.append(self._recv_from_tokenizer.get()) + while not self._recv_from_tokenizer.empty(): + pending_msgs.append(self._recv_from_tokenizer.get()) + return pending_msgs + + def _recv_msg_multi_rank0(self, blocking: bool = False) -> List[BaseBackendMsg]: + pending_msgs: List[BaseBackendMsg] = [] + if blocking: + self.run_when_idle() + raw = self._recv_from_tokenizer.get_raw() + self._send_into_ranks.put_raw(raw) + pending_msgs.append(self._recv_from_tokenizer.decode(raw)) + + pending_raw_msgs: List[bytes] = [] + while not self._recv_from_tokenizer.empty(): + pending_raw_msgs.append(self._recv_from_tokenizer.get_raw()) + + # broadcast the number of raw messages to all ranks + src_tensor = torch.tensor(len(pending_raw_msgs)) + self.tp_cpu_group.broadcast(src_tensor, root=0).wait() + + for raw in pending_raw_msgs: + self._send_into_ranks.put_raw(raw) + pending_msgs.append(self._recv_from_tokenizer.decode(raw)) + return pending_msgs + + def _recv_msg_multi_rank1(self, blocking: bool = False) -> List[BaseBackendMsg]: + pending_msgs: List[BaseBackendMsg] = [] + if blocking: + self.run_when_idle() + pending_msgs.append(self._recv_from_rank0.get()) + + # ensure all ranks have the same number of raw messages + dst_tensor = torch.tensor(-1) + self.tp_cpu_group.broadcast(dst_tensor, root=0).wait() + dst_length = int(dst_tensor.item()) + + for _ in range(dst_length): + pending_msgs.append(self._recv_from_rank0.get()) + return pending_msgs + + def _reply_tokenizer_rank0(self, reply: BatchTokenizerMsg) -> None: + num_reply = len(reply.data) + logger.debug_rank0(f"Replying to tokenizer: {num_reply} messages") + if num_reply == 1: + self._send_into_tokenizer.put(reply.data[0]) + elif num_reply > 1: + self._send_into_tokenizer.put(reply) + + def _reply_tokenizer_rank1(self, reply: BatchTokenizerMsg) -> None: + _ = reply # do nothing for non-primary ranks diff --git a/zonos2/scheduler/scheduler.py b/zonos2/scheduler/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..59ca49301329e7097a33b08370b55d647a942fe4 --- /dev/null +++ b/zonos2/scheduler/scheduler.py @@ -0,0 +1,796 @@ +"""TTS Scheduler for multi-codebook audio generation.""" + +from __future__ import annotations + +import logging +import time +from dataclasses import replace +from typing import TYPE_CHECKING, List, NamedTuple, NoReturn, Set, Tuple, TypeAlias + +import torch +import torch.nn.functional as F +from zonos2.core import TTSBatch, TTSReq +from zonos2.env import ENV +from zonos2.message import ( + BaseTTSBackendMsg, + BatchTTSBackendMsg, + BatchTTSTokenizerMsg, + ExitMsg, + TTSDetokenizeMsg, + TTSUserMsg, +) +from zonos2.utils import init_logger + +from .cache import CacheManager +from .config import SchedulerConfig +from .decode import TTSDecodeManager +from .io import SchedulerIOMixin +from .table import TTSTableManager + +if TYPE_CHECKING: + from zonos2.engine.sample import TTSBatchSamplingArgs + +logger = init_logger(__name__) + + +class TTSForwardOutput(NamedTuple): + """Output from TTS forward pass.""" + + next_tokens_gpu: torch.Tensor # (batch, frame_width) + next_tokens_cpu: torch.Tensor # (batch, frame_width) + copy_done: torch.cuda.Event + + +class TTSForwardInput(NamedTuple): + """Input for TTS forward pass.""" + + batch: TTSBatch + sample_args: TTSBatchSamplingArgs + load_indices: torch.Tensor + write_indices: torch.Tensor + + +TTSForwardData: TypeAlias = "Tuple[TTSForwardInput, TTSForwardOutput]" + + +def _make_3d_indices(table_3d: torch.Tensor, ranges: List[Tuple[int, int, int]]) -> torch.Tensor: + """Return 1D indices for a 3D token pool. + + Args: + table_3d: 3D tensor (max_reqs, max_seq_len, frame_width) + ranges: List of (req_idx, begin, end) tuples + + Returns: + 1D tensor of indices into the flattened token pool. + These indices are designed to be divided by frame_width to get + row indices into flat_pool.view(-1, frame_width). + """ + assert table_3d.dim() == 3 and table_3d.is_contiguous() + req_stride, seq_stride, _ = table_3d.stride() + # For shape (max_reqs, max_seq_len, frame_width): + # req_stride = max_seq_len * frame_width + # seq_stride = frame_width + # We compute: entry * req_stride + pos * seq_stride + # When divided by frame_width, this gives: entry * max_seq_len + pos + needed_size = sum(end - begin for _, begin, end in ranges) + indices_host = torch.empty(needed_size, dtype=torch.int32, pin_memory=True) + offset = 0 + for entry, begin, end in ranges: + length = end - begin + base = entry * req_stride + for i in range(length): + indices_host[offset + i] = base + (begin + i) * seq_stride + offset += length + return indices_host.to(table_3d.device, non_blocking=True) + + +def _make_speaker_slot( + input_ids: torch.Tensor, + *, + n_codebooks: int, + audio_pad_id: int, + text_vocab: int, +) -> torch.Tensor: + """Create a reserved frame for speaker embedding injection at position 0.""" + assert input_ids.dim() == 2 + slot = torch.zeros( + (1, input_ids.shape[-1]), + dtype=input_ids.dtype, + device=input_ids.device, + ) + audio_cols = min(n_codebooks, input_ids.shape[-1]) + slot[:, :audio_cols] = int(audio_pad_id) + if input_ids.shape[-1] > n_codebooks: + text_pad = int(text_vocab) if int(text_vocab) > 0 else int(audio_pad_id) + slot[:, n_codebooks] = text_pad + return slot + + +def _has_reserved_speaker_slot( + input_ids: torch.Tensor, + *, + n_codebooks: int, + audio_pad_id: int, + text_vocab: int, +) -> bool: + """Check whether the first frame already matches the reserved speaker slot.""" + if input_ids.shape[0] == 0 or input_ids.shape[-1] <= n_codebooks: + return False + first = input_ids[0] + audio_cols = min(n_codebooks, input_ids.shape[-1]) + audio_ok = bool(torch.all(first[:audio_cols] == int(audio_pad_id))) + text_expected = int(text_vocab) if int(text_vocab) > 0 else int(audio_pad_id) + text_ok = int(first[n_codebooks].item()) == text_expected + return audio_ok and text_ok + + +def _make_marker_slot( + input_ids: torch.Tensor, + *, + n_codebooks: int, + audio_pad_id: int, + text_token: int, +) -> torch.Tensor: + """Create a conditioning marker frame: audio padding plus one text token.""" + assert input_ids.dim() == 2 + slot = torch.zeros( + (1, input_ids.shape[-1]), + dtype=input_ids.dtype, + device=input_ids.device, + ) + audio_cols = min(n_codebooks, input_ids.shape[-1]) + slot[:, :audio_cols] = int(audio_pad_id) + if input_ids.shape[-1] > n_codebooks: + slot[:, n_codebooks] = int(text_token) + return slot + + +def _frame_text_token( + input_ids: torch.Tensor, + frame_idx: int, + *, + n_codebooks: int, + audio_pad_id: int, +) -> int | None: + """Return the text token of an audio-padded frame, or None if not a marker.""" + if input_ids.shape[0] <= frame_idx or input_ids.shape[-1] <= n_codebooks: + return None + frame = input_ids[frame_idx] + audio_cols = min(n_codebooks, input_ids.shape[-1]) + if not bool(torch.all(frame[:audio_cols] == int(audio_pad_id))): + return None + return int(frame[n_codebooks].item()) + + +class TTSScheduler(SchedulerIOMixin): + """Scheduler for TTS multi-codebook generation.""" + + def __init__(self, config: SchedulerConfig): + from zonos2.engine import Engine + from zonos2.engine.sample import TTSSampler + + self.engine = Engine(config) + super().__init__(config, self.engine.tp_cpu_group) + + self.device = self.engine.device + self.stream = torch.cuda.Stream(device=self.device) + self.engine_stream_ctx = torch.cuda.stream(self.engine.stream) + torch.cuda.set_stream(self.stream) + + # TTS-specific configuration - read from model config + model_config = self.engine.model_config + self.n_codebooks = model_config.n_codebooks + self.codebook_size = model_config.codebook_size + self.eoa_id = model_config.eoa_id + self.audio_pad_id = model_config.audio_pad_id + self.text_vocab = model_config.text_vocab if model_config.text_vocab is not None else 0 + self.speaker_enabled = bool(getattr(model_config, "speaker_enabled", False)) + self.speaker_embedding_dim = int(getattr(model_config, "speaker_embedding_dim", 0)) + self.speaker_background_token_enabled = bool( + getattr(model_config, "speaker_background_token_enabled", False) + ) + self.accurate_mode_token_enabled = bool( + getattr(model_config, "accurate_mode_token_enabled", False) + ) + self.speaker_background_num_buckets = 2 if self.speaker_background_token_enabled else 0 + self.accurate_mode_num_buckets = ( + 1 + if self.accurate_mode_token_enabled and self.speaker_background_num_buckets > 0 + else 0 + ) + self.speaking_rate_num_buckets = int(getattr(model_config, "speaking_rate_num_buckets", 0)) + quality_buckets = getattr(model_config, "quality_buckets", None) or {} + quality_features = tuple(getattr(model_config, "quality_features", ()) or ()) + if not quality_features: + quality_features = tuple(quality_buckets.keys()) + self.quality_features = quality_features + self.quality_bucket_counts = [ + len(quality_buckets.get(feature, ())) for feature in quality_features + ] + # Frame width includes n_codebooks audio codes and one text token. + self.frame_width = self.n_codebooks + 1 + + # TTS sampler + self.tts_sampler = TTSSampler( + device=self.device, + n_codebooks=self.n_codebooks, + codebook_size=self.codebook_size, + text_vocab=self.text_vocab, + ) + + # Initialize TTS-specific managers + self.table_manager = TTSTableManager( + config.max_running_req, self.engine.page_table, self.frame_width + ) + self.cache_manager = CacheManager(self.device, self.engine.num_pages, config.cache_type) + self.decode_manager = TTSDecodeManager() + + # Waiting queue for prefill + self.waiting_reqs: List[TTSUserMsg] = [] + + self.tp_info = config.tp_info + self.finished_reqs: Set[TTSReq] = set() + self.page_table = self.engine.page_table + self.token_pool = self.table_manager.token_pool + self.prefill_budget = config.max_extend_tokens + + # Metrics tracking for periodic throughput logging + self._stats_start = time.perf_counter() + self._stats_batches = 0 + self._stats_tokens = 0 + + def _process_last_data( + self, + last_data: TTSForwardData | None, + ongoing_data: TTSForwardData | None, + ) -> None: + if last_data is None: + return + + batch, (_, next_tokens_cpu, copy_done) = last_data[0].batch, last_data[1] + copy_done.synchronize() + reply = BatchTTSTokenizerMsg(data=[]) + logger.debug( + "Processing batch: size=%d, padded_size=%d, next_tokens shape=%s", + batch.size, + batch.padded_size, + next_tokens_cpu.shape, + ) + + max_seq_len = self.engine.max_seq_len + for i, req in enumerate(batch.reqs): + if req in self.finished_reqs: + continue + + # Get audio codes for this frame + next_token = next_tokens_cpu[i] # (frame_width,) + audio_codes = next_token[: self.n_codebooks].tolist() + + # Append to host storage + req.append_host(next_token) + + # Log progress + generated = req.total_generated + if generated <= 5: + logger.debug("TTS %d frame %d: audio_codes=%s", req.uid, generated, audio_codes) + + # Check for EOS + finished = req.remain_len <= 0 + if req.check_eos(audio_codes): + finished = True + logger.debug( + "TTS %d EOS detected at frame %d, eos_frame=%d", + req.uid, + generated, + req.eos_frame, + ) + + if req.device_len >= max_seq_len - 1: + finished = True + logger.warning_rank0(f"TTS request {req.uid} reached {max_seq_len = }, dropped.") + + # Send detokenize message with audio codes + reply.data.append( + TTSDetokenizeMsg( + uid=req.uid, + audio_codes=audio_codes, + finished=finished, + eos_frame=req.eos_frame if req.eos_frame >= 0 else None, + ) + ) + + if finished: + self.finished_reqs.add(req) + self.decode_manager.remove_req(req) + logger.debug("TTS %d finished: %d frames generated", req.uid, generated) + + # Free resources for finished but not ongoing reqs + ongoing_reqs = ongoing_data[0].batch.reqs if ongoing_data else [] + for req in self.finished_reqs.difference(ongoing_reqs): + # Free the allocated cache slots (cached_len represents actually allocated positions) + slots = self.page_table[req.table_idx, : req.cached_len] + self.cache_manager.free_slots(slots) + self.cache_manager.free_handle(req.cache_handle) + self.table_manager.free(req.table_idx) + + self.finished_reqs.intersection_update(ongoing_reqs) + self.send_result(reply) + + def _process_one_msg(self, msg: BaseTTSBackendMsg) -> None: + if isinstance(msg, BatchTTSBackendMsg): + for m in msg.data: + self._process_one_msg(m) + elif isinstance(msg, ExitMsg): + raise KeyboardInterrupt + elif isinstance(msg, TTSUserMsg): + logger.debug_rank0("Received TTS user msg: %s", msg.uid) + input_len = len(msg.input_ids) + max_seq_len = self.engine.max_seq_len + if input_len >= max_seq_len: + return logger.warning_rank0( + f"TTS input len {input_len} exceeds {max_seq_len}, " + f"request {msg.uid} is dropped." + ) + self.waiting_reqs.append(msg) + else: + logger.error(f"Unknown TTS message type: {type(msg)}") + raise NotImplementedError + + def _speaker_background_token(self, clean: bool) -> int: + # Imported lazily: zonos2.tts pulls in TTSLLM, which imports this module. + from zonos2.tts.prompt import speaker_background_token_id + + return speaker_background_token_id( + self.text_vocab, + self.speaking_rate_num_buckets, + self.quality_bucket_counts, + clean, + self.speaker_background_num_buckets, + self.accurate_mode_num_buckets, + ) + + def _accurate_mode_token(self) -> int: + from zonos2.tts.prompt import accurate_mode_token_id + + return accurate_mode_token_id( + self.text_vocab, + self.speaking_rate_num_buckets, + self.quality_bucket_counts, + self.speaker_background_num_buckets, + self.accurate_mode_num_buckets, + ) + + def _with_speaker_frames( + self, + input_ids: torch.Tensor, + msg: TTSUserMsg, + speaker_token_position: int, + ) -> torch.Tensor: + """Ensure the canonical speaker slot and marker frames lead the prompt. + + Train-time prompt layout: speaker slot at frame 0, then the clean/noisy + speaker-background marker, then (optionally) the accurate-mode marker. + """ + needs_background = self.speaker_background_num_buckets > 0 + needs_accurate = self.accurate_mode_num_buckets > 0 and bool(msg.accurate_mode) + + def _marker(text_token: int) -> torch.Tensor: + return _make_marker_slot( + input_ids, + n_codebooks=self.n_codebooks, + audio_pad_id=self.audio_pad_id, + text_token=text_token, + ) + + has_slot = speaker_token_position == 0 and _has_reserved_speaker_slot( + input_ids, + n_codebooks=self.n_codebooks, + audio_pad_id=self.audio_pad_id, + text_vocab=self.text_vocab, + ) + if not has_slot: + prefix = [ + _make_speaker_slot( + input_ids, + n_codebooks=self.n_codebooks, + audio_pad_id=self.audio_pad_id, + text_vocab=self.text_vocab, + ) + ] + if needs_background: + prefix.append(_marker(self._speaker_background_token(msg.clean_speaker_background))) + if needs_accurate: + prefix.append(_marker(self._accurate_mode_token())) + return torch.cat(prefix + [input_ids], dim=0) + + if not needs_background: + return input_ids + + background_ids = { + self._speaker_background_token(True), + self._speaker_background_token(False), + } + frame1 = _frame_text_token( + input_ids, 1, n_codebooks=self.n_codebooks, audio_pad_id=self.audio_pad_id + ) + if frame1 not in background_ids: + inserted = [_marker(self._speaker_background_token(msg.clean_speaker_background))] + if needs_accurate: + inserted.append(_marker(self._accurate_mode_token())) + return torch.cat([input_ids[:1]] + inserted + [input_ids[1:]], dim=0) + + if needs_accurate: + frame2 = _frame_text_token( + input_ids, 2, n_codebooks=self.n_codebooks, audio_pad_id=self.audio_pad_id + ) + if frame2 != self._accurate_mode_token(): + return torch.cat( + [input_ids[:2], _marker(self._accurate_mode_token()), input_ids[2:]], dim=0 + ) + + return input_ids + + def _schedule_prefill(self) -> TTSBatch | None: + """Schedule a prefill batch from waiting requests.""" + if not self.waiting_reqs or self.table_manager.available_size == 0: + return None + + reqs = [] + total_tokens = 0 + + while self.waiting_reqs and self.table_manager.available_size > 0: + if total_tokens >= self.prefill_budget: + break + + msg = self.waiting_reqs[0] + input_ids = msg.input_ids + speaker_token_position = msg.speaker_token_position + if msg.speaker_embedding is not None: + input_ids = self._with_speaker_frames(input_ids, msg, speaker_token_position) + speaker_token_position = 0 + + input_len = len(input_ids) + max_seq_len = self.engine.max_seq_len + if input_len >= max_seq_len: + logger.warning_rank0( + f"TTS input len {input_len} exceeds {max_seq_len}, " + f"request {msg.uid} is dropped." + ) + self.waiting_reqs.pop(0) + continue + + max_output_len = max_seq_len - input_len + sampling_params = msg.sampling_params + if msg.sampling_params.max_tokens > max_output_len: + sampling_params = replace(msg.sampling_params, max_tokens=max_output_len) + logger.debug_rank0( + "Adjust TTS max_tokens to %d for request %d.", + max_output_len, + msg.uid, + ) + + if total_tokens + input_len > self.prefill_budget and reqs: + break # Don't exceed budget unless this is the first request + + self.waiting_reqs.pop(0) + total_tokens += input_len + + # Create TTSReq + table_idx = self.table_manager.allocate() + cache_handle = self.cache_manager.allocate_new_handle() + + # Copy input_ids to token pool + input_ids_i32 = input_ids.to(torch.int32) + self.token_pool[table_idx, :input_len].copy_( + input_ids_i32.pin_memory().to(self.device), non_blocking=True + ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "TTS %d prefill: input_len=%d, input_ids shape=%s", + msg.uid, + input_len, + input_ids_i32.shape, + ) + logger.debug("TTS %d first frame: %s", msg.uid, input_ids_i32[0].tolist()) + logger.debug("TTS %d last frame: %s", msg.uid, input_ids_i32[-1].tolist()) + if input_len > 20: + logger.debug("TTS %d frame at -20: %s", msg.uid, input_ids_i32[-20].tolist()) + logger.debug("TTS %d frame at -17: %s", msg.uid, input_ids_i32[-17].tolist()) + + rng = None + if msg.sampling_params.seed is not None: + rng = torch.Generator(device=self.device) + rng.manual_seed(msg.sampling_params.seed) + + req = TTSReq( + input_ids=input_ids_i32, + table_idx=table_idx, + cached_len=0, + output_len=sampling_params.max_tokens, + uid=msg.uid, + sampling_params=sampling_params, + cache_handle=cache_handle, + n_codebooks=self.n_codebooks, + eoa_id=self.eoa_id, + rng=rng, + speaker_embedding=msg.speaker_embedding, + speaker_token_position=speaker_token_position, + ) + reqs.append(req) + + if not reqs: + return None + + return TTSBatch(reqs=reqs, phase="prefill") + + def _prepare_batch(self, batch: TTSBatch) -> TTSForwardInput: + """Prepare a batch for forward pass.""" + needed_size = sum(r.extend_len for r in batch.reqs) + batch.out_loc = self.cache_manager.allocate(needed_size) + + # Pad batch if needed for CUDA graph + if padding_size := self.engine.graph_runner.pad_batch(batch): + batch.out_loc = F.pad(batch.out_loc, (0, padding_size), value=self.engine.dummy_page) + + # For TTS we load full frames + load_ranges = [(r.table_idx, r.cached_len, r.device_len) for r in batch.padded_reqs] + write_ranges = [(r.table_idx, r.device_len, r.device_len + 1) for r in batch.reqs] + + load_indices = _make_3d_indices(self.token_pool, load_ranges) + write_indices = _make_3d_indices(self.token_pool, write_ranges) + + # Write out_loc to page table + self.page_table.view(-1)[load_indices // self.frame_width] = batch.out_loc + self.engine.attn_backend.prepare_metadata(batch) + + return TTSForwardInput( + batch=batch, + sample_args=self.tts_sampler.prepare(batch, token_pool=self.token_pool), + load_indices=load_indices, + write_indices=write_indices, + ) + + def _schedule_next_batch(self) -> TTSForwardInput | None: + """Schedule next batch (prefill first, then decode).""" + batch = self._schedule_prefill() or self.decode_manager.schedule_next_batch() + return self._prepare_batch(batch) if batch else None + + def _log_batch_stats(self, batch: TTSBatch) -> None: + self._stats_batches += 1 + self._stats_tokens += batch.size + + if batch.is_prefill: + logger.info_rank0( + "TTS Prefill: reqs=%d, extend_frames=%d, waiting=%d, decode_running=%d", + batch.size, + sum(r.extend_len for r in batch.reqs), + len(self.waiting_reqs), + len(self.decode_manager.running_reqs), + ) + + now = time.perf_counter() + elapsed = now - self._stats_start + if elapsed >= 5.0: + logger.info_rank0( + "TTS Throughput (%.1fs window): batches=%d, decode_steps=%d, %.1f frames/s, " + "decode_running=%d, waiting=%d", + elapsed, + self._stats_batches, + self._stats_tokens, + self._stats_tokens / elapsed, + len(self.decode_manager.running_reqs), + len(self.waiting_reqs), + ) + self._stats_start = now + self._stats_batches = 0 + self._stats_tokens = 0 + + def _load_token_ids(self, input: TTSForwardInput) -> None: + """Load token IDs from pool to batch.""" + # Load full frames for TTS + flat_pool = self.token_pool.view(-1, self.frame_width) + frame_indices = input.load_indices // self.frame_width + input.batch.input_ids = flat_pool[frame_indices] + + # Debug: verify loaded tokens + if logger.isEnabledFor(logging.DEBUG) and input.batch.is_decode and input.batch.size > 0: + req = input.batch.reqs[0] + if req.total_generated < 10: + loaded_tokens = input.batch.input_ids[0].tolist() + logger.debug( + "Load frame %d: loaded from frame_index %d, tokens = %s", + req.total_generated, + frame_indices[0].item(), + loaded_tokens, + ) + + def _write_token_ids(self, input: TTSForwardInput, output: TTSForwardOutput) -> None: + """Write generated tokens back to pool.""" + flat_pool = self.token_pool.view(-1, self.frame_width) + frame_indices = input.write_indices // self.frame_width + flat_pool[frame_indices] = output.next_tokens_gpu + + # Debug: verify what was written + if logger.isEnabledFor(logging.DEBUG) and input.batch.size > 0: + req = input.batch.reqs[0] + if req.total_generated < 10: + written_tokens = flat_pool[frame_indices[0]].tolist() + logger.debug( + "Write frame %d: wrote tokens %s at frame_index %d", + req.total_generated, + written_tokens, + frame_indices[0].item(), + ) + + def _prepare_speaker_conditioning(self, batch: TTSBatch) -> None: + """Prepare per-batch speaker embeddings and injection positions.""" + if not self.speaker_enabled: + batch.speaker_emb_values = None + batch.speaker_token_positions = None + return + + emb_values: list[torch.Tensor] = [] + token_positions: list[int] = [] + offset = 0 + + for req in batch.reqs: + if req.speaker_embedding is None: + offset += req.extend_len + continue + + rel_pos = req.speaker_token_position - req.cached_len + if 0 <= rel_pos < req.extend_len: + emb = req.speaker_embedding + if emb.numel() != self.speaker_embedding_dim: + logger.warning_rank0( + "Skipping speaker embedding for uid=%d: expected dim=%d, got=%d", + req.uid, + self.speaker_embedding_dim, + emb.numel(), + ) + else: + emb_values.append(emb.to(torch.float32)) + token_positions.append(offset + rel_pos) + offset += req.extend_len + + if emb_values: + batch.speaker_emb_values = torch.stack(emb_values, dim=0).to( + self.device, non_blocking=True + ) + batch.speaker_token_positions = torch.tensor( + token_positions, dtype=torch.long, device=self.device + ) + else: + batch.speaker_emb_values = None + batch.speaker_token_positions = None + + def _forward(self, forward_input: TTSForwardInput) -> TTSForwardOutput: + """Run forward pass on batch.""" + self._load_token_ids(forward_input) + batch, sample_args = forward_input.batch, forward_input.sample_args + self._prepare_speaker_conditioning(batch) + + # Debug: show input shape and first request's input + if logger.isEnabledFor(logging.DEBUG): + if batch.is_prefill and batch.size > 0: + logger.debug("Forward prefill input_ids shape: %s", batch.input_ids.shape) + elif batch.is_decode and batch.size > 0: + req = batch.reqs[0] + if req.total_generated < 10: + input_tokens = batch.input_ids[0].tolist() + eoa_id = self.eoa_id + eoa_positions = [ + i for i, t in enumerate(input_tokens[: self.n_codebooks]) if t == eoa_id + ] + logger.debug( + "Forward decode step %d: input tokens = %s", + req.total_generated, + input_tokens, + ) + if eoa_positions: + logger.debug( + "Forward WARNING: EOA token in INPUT at codebook positions %s!", + eoa_positions, + ) + positions = batch.attn_metadata.positions + logger.debug( + "Forward positions: %s", + ( + positions[: batch.size].tolist() + if hasattr(positions, "tolist") + else positions + ), + ) + + # Forward through engine - get multi-codebook logits + logits = self.engine.forward_batch_tts(batch) + + # Debug: check logits stats for first 5 frames + if logger.isEnabledFor(logging.DEBUG) and batch.size > 0: + req = batch.reqs[0] + if req.total_generated < 5: + logger.debug( + "Forward frame %d: logits shape=%s, min=%.2f, max=%.2f, mean=%.2f", + req.total_generated, + logits.shape, + logits.min().item(), + logits.max().item(), + logits.mean().item(), + ) + + # Sample tokens + next_tokens = self.tts_sampler.sample_to_tensor(logits, sample_args) + + # Copy to CPU + next_tokens_cpu = torch.empty_like(next_tokens, device="cpu", pin_memory=True) + copy_done = torch.cuda.Event() + next_tokens_cpu.copy_(next_tokens, non_blocking=True) + copy_done.record() + + self._write_token_ids( + forward_input, + TTSForwardOutput(next_tokens, next_tokens_cpu, copy_done), + ) + + # Mark all requests as available for decode + for req in batch.reqs: + req.complete_one() + self.decode_manager.add_reqs(forward_input.batch.reqs) + + return TTSForwardOutput(next_tokens, next_tokens_cpu, copy_done) + + def run_when_idle(self) -> None: + """Called when scheduler is idle.""" + logger.info_rank0("TTS Scheduler is idle, waiting for new reqs...") + self.cache_manager.check_integrity() + + def overlap_loop(self, last_data: TTSForwardData | None) -> TTSForwardData | None: + """Main loop with overlapping scheduling and execution.""" + blocking = not (last_data or self.waiting_reqs or self.decode_manager.runnable) + for msg in self.receive_msg(blocking=blocking): + self._process_one_msg(msg) + + forward_input = self._schedule_next_batch() + ongoing_data = None + if forward_input is not None: + self._log_batch_stats(forward_input.batch) + with self.engine_stream_ctx: + self.engine.stream.wait_stream(self.stream) + ongoing_data = (forward_input, self._forward(forward_input)) + + self._process_last_data(last_data, ongoing_data) + return ongoing_data + + def normal_loop(self) -> None: + """Non-overlapping loop.""" + blocking = not (self.waiting_reqs or self.decode_manager.runnable) + for msg in self.receive_msg(blocking=blocking): + self._process_one_msg(msg) + + forward_input = self._schedule_next_batch() + ongoing_data = None + if forward_input is not None: + self._log_batch_stats(forward_input.batch) + ongoing_data = (forward_input, self._forward(forward_input)) + + self._process_last_data(ongoing_data, None) + + @torch.inference_mode() + def run_forever(self) -> NoReturn: + """Run scheduler loop forever.""" + if ENV.DISABLE_OVERLAP_SCHEDULING: + with self.engine_stream_ctx: + self.engine.stream.wait_stream(self.stream) + while True: + self.normal_loop() + else: + assert torch.cuda.current_stream() == self.stream + data = None + while True: + data = self.overlap_loop(data) + + def shutdown(self) -> None: + """Shutdown the scheduler.""" + torch.cuda.synchronize(self.device) + self.sync_all_ranks() + self.engine.shutdown() diff --git a/zonos2/scheduler/table.py b/zonos2/scheduler/table.py new file mode 100644 index 0000000000000000000000000000000000000000..50edfc7c1312d52709fe404a9da1178d35d5c16b --- /dev/null +++ b/zonos2/scheduler/table.py @@ -0,0 +1,48 @@ +"""TTS table manager for 2D token pool.""" + +from __future__ import annotations + +import torch + + +class TTSTableManager: + """Table manager for TTS with 3D token pool (max_reqs, max_seq_len, frame_width).""" + + def __init__( + self, + max_running_reqs: int, + page_table: torch.Tensor, + frame_width: int, + ) -> None: + """Initialize the TTS table manager. + + Args: + max_running_reqs: Maximum number of concurrent requests + page_table: Page table tensor for KV cache management + frame_width: Number of elements per frame (n_codebooks + extras) + """ + self._max_running_reqs = max_running_reqs + self._free_slots = list(range(max_running_reqs)) + self.page_table = page_table + self.frame_width = frame_width + + # 3D token pool: (max_reqs, max_seq_len, frame_width) + # Initialized with zeros (valid padding) + self.token_pool = torch.zeros( + (page_table.shape[0], page_table.shape[1], frame_width), + dtype=torch.int32, + device=page_table.device, + ) + + @property + def available_size(self) -> int: + """Number of available request slots.""" + return len(self._free_slots) + + def allocate(self) -> int: + """Allocate a table slot for a new request.""" + return self._free_slots.pop() + + def free(self, slot: int) -> None: + """Free a table slot.""" + self._free_slots.append(slot) diff --git a/zonos2/server/__init__.py b/zonos2/server/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa9b4736095d6e0a2ede77df7fe08847c625fb8a --- /dev/null +++ b/zonos2/server/__init__.py @@ -0,0 +1,3 @@ +from .launch import launch_server + +__all__ = ["launch_server"] diff --git a/zonos2/server/api_server.py b/zonos2/server/api_server.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ea26acf3d203083785036d0d0dadb78de34837 --- /dev/null +++ b/zonos2/server/api_server.py @@ -0,0 +1,1990 @@ +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import io +import math +import os +import re +import subprocess +import time +import uuid +import wave +from collections.abc import Mapping +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, List, Literal, Tuple + +import numpy as np +import torch +import uvicorn +from fastapi import FastAPI, Header, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, Response, StreamingResponse +from pydantic import BaseModel, Field +from starlette.background import BackgroundTask +from zonos2.message import ( + BaseFrontendMsg, + BaseTokenizerMsg, + BatchTTSFrontendMsg, + TTSAudioReply, + TTSSamplingParams, + TTSTokenizeMsg, +) +from zonos2.utils import ZmqAsyncPullQueue, ZmqAsyncPushQueue, init_logger + +from .args import ServerArgs + +_UI_HTML = Path(__file__).resolve().parent.parent.parent.parent / "tts_ui.html" + +logger = init_logger(__name__, "FrontendAPI") + +_GLOBAL_STATE = None +_SPEAKER_EMBEDDERS = {} +_SPEAKER_EMBEDDER_LOCK = asyncio.Lock() +_SPEAKER_EMBEDDER_DEVICE = os.getenv("ZONOS2_SPEAKER_EMBEDDER_DEVICE", "cpu") +_SESSION_SPEAKER_CACHE: Dict[str, Dict[str, "CachedSpeakerReference"]] = {} +_SESSION_SPEAKER_CACHE_LOCK = asyncio.Lock() +_DEFAULT_SPEAKER_CACHE: Dict[Tuple[str, int], Dict[str, "DefaultSpeakerReference"]] = {} +_DEFAULT_SPEAKER_CACHE_LOCK = asyncio.Lock() +_SPEAKING_RATE_FPS = 86.0 * (44070.0 / 44000.0) +_DEFAULT_SPEAKING_RATE_BYTES_PER_SECOND = 15.0 +_SPEAKING_RATE_CLOSED_BUCKET_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)\s*$") +_SPEAKING_RATE_OPEN_BUCKET_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*\+\s*$") +_QUALITY_NUMBER_RE = r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)" +_QUALITY_EXACT_BUCKET_RE = re.compile(rf"^\s*({_QUALITY_NUMBER_RE})\s*$") +_QUALITY_CLOSED_BUCKET_RE = re.compile( + rf"^\s*({_QUALITY_NUMBER_RE})\s*-\s*({_QUALITY_NUMBER_RE})\s*$" +) +_QUALITY_OPEN_BUCKET_RE = re.compile(rf"^\s*({_QUALITY_NUMBER_RE})\s*\+\s*$") +_QUALITY_METRIC_FIELDS = ( + "lufs", + "estimated_snr", + "max_pause", + "estimated_bandlimit_hz", + "leading_silence_s", + "trailing_silence_s", +) +_DEFAULT_QUALITY_BUCKETS = {"trailing_silence_s": 3} +_DEFAULT_VOICE_AUDIO_EXTENSIONS = { + ".aac", + ".flac", + ".m4a", + ".mp3", + ".ogg", + ".opus", + ".wav", + ".webm", +} +_DEFAULT_VOICE_EMBEDDING_EXTENSIONS = {".npy", ".npz"} + + +@dataclass +class CachedSpeakerReference: + speaker_id: str + label: str + source_type: Literal["audio", "embedding_file"] + embedding: torch.Tensor + created_at: float + original_name: str | None = None + audio_bytes: bytes | None = None + audio_media_type: str | None = None + + +@dataclass +class DefaultSpeakerReference: + speaker_id: str + label: str + source_type: Literal["audio", "embedding_file"] + path: Path + mtime: float + original_name: str + embedding: torch.Tensor | None = None + audio_bytes: bytes | None = None + audio_media_type: str | None = None + + +def get_global_state() -> FrontendManager: + global _GLOBAL_STATE + assert _GLOBAL_STATE is not None, "Global state is not initialized" + return _GLOBAL_STATE + + +def _unwrap_msg(msg: BaseFrontendMsg) -> List[TTSAudioReply]: + if isinstance(msg, BatchTTSFrontendMsg): + result = [] + for reply in msg.data: + assert isinstance(reply, TTSAudioReply) + result.append(reply) + return result + if isinstance(msg, TTSAudioReply): + return [msg] + raise TypeError(f"Unexpected frontend message type: {type(msg).__name__}") + + +def _model_supports_speaker(config: ServerArgs) -> bool: + return bool(getattr(config.model_config, "speaker_enabled", False)) + + +def _model_speaker_dim(config: ServerArgs) -> int: + return int(getattr(config.model_config, "speaker_embedding_dim", 128)) + + +def _model_speaking_rate_num_buckets(config: ServerArgs) -> int: + model_value = int(getattr(config.model_config, "speaking_rate_num_buckets", 0) or 0) + server_value = int(getattr(config, "tts_speaking_rate_num_buckets", 0) or 0) + return model_value or server_value + + +def _model_speaking_rate_buckets(config: ServerArgs) -> list[str]: + raw = getattr(config.model_config, "speaking_rate_buckets", None) + if not raw: + raw = getattr(config, "tts_speaking_rate_buckets", ()) + return [str(item) for item in (raw or ())] + + +def _model_tts_max_tokens(config: ServerArgs) -> int: + return max(1, int(config.max_seq_len)) + + +def _resolve_tts_max_tokens(config: ServerArgs, requested: int | None) -> int: + model_max = _model_tts_max_tokens(config) + if requested is None: + return model_max + requested = int(requested) + if requested <= 0: + raise ValueError("max_tokens must be positive.") + return min(requested, model_max) + + +def _field_was_set(model: BaseModel, field_name: str) -> bool: + fields_set = getattr(model, "model_fields_set", None) + if fields_set is None: + fields_set = getattr(model, "__fields_set__", set()) + return field_name in fields_set + + +def _parse_speaking_rate_bucket(spec: str) -> tuple[float, float | None]: + closed = _SPEAKING_RATE_CLOSED_BUCKET_RE.match(str(spec)) + if closed is not None: + return float(closed.group(1)), float(closed.group(2)) + + open_ended = _SPEAKING_RATE_OPEN_BUCKET_RE.match(str(spec)) + if open_ended is not None: + return float(open_ended.group(1)), None + + raise ValueError(f"Invalid speaking-rate bucket {spec!r}; expected ranges like '0-3' or '60+'.") + + +def _speaking_rate_bucket_ranges(config: ServerArgs) -> list[tuple[float, float | None]]: + ranges = [_parse_speaking_rate_bucket(spec) for spec in _model_speaking_rate_buckets(config)] + if not ranges: + return ranges + + first_low, _ = ranges[0] + if not math.isclose(first_low, 0.0, abs_tol=1e-9): + raise ValueError("speaking-rate buckets must start at 0.") + + previous_high: float | None = None + for idx, (low, high) in enumerate(ranges): + if low < 0.0: + raise ValueError("speaking-rate buckets must use non-negative ranges.") + if high is not None and high <= low: + raise ValueError(f"speaking-rate bucket {idx} has an empty or inverted range.") + if previous_high is None and idx > 0: + raise ValueError( + "speaking-rate buckets cannot define ranges after an open-ended bucket." + ) + if previous_high is not None and not math.isclose(low, previous_high, abs_tol=1e-9): + raise ValueError("speaking-rate buckets must be contiguous and ordered.") + previous_high = high + + if ranges[-1][1] is not None: + raise ValueError("speaking-rate buckets must end with an open-ended range like '60+'.") + return ranges + + +def _speaking_rate_bucket_for_rate( + rate_bytes_per_second: float, + *, + num_buckets: int, + ranges: list[tuple[float, float | None]], +) -> int: + if rate_bytes_per_second <= 0: + raise ValueError("speaking_rate must be positive.") + + if ranges: + for idx, (_, high) in enumerate(ranges): + if high is None or ( + rate_bytes_per_second < high + and not math.isclose(rate_bytes_per_second, high, rel_tol=1e-12, abs_tol=1e-9) + ): + return idx + return len(ranges) - 1 + + rate_bytes_per_frame = rate_bytes_per_second / _SPEAKING_RATE_FPS + bucket = int(rate_bytes_per_frame * num_buckets) + return min(max(bucket, 0), num_buckets - 1) + + +def _neutral_speaking_rate_bytes_per_second( + ranges: list[tuple[float, float | None]], +) -> float: + if not ranges: + return _DEFAULT_SPEAKING_RATE_BYTES_PER_SECOND + + low, high = ranges[len(ranges) // 2] + if high is None: + return max(low, _DEFAULT_SPEAKING_RATE_BYTES_PER_SECOND) + return (low + high) / 2.0 + + +def _resolve_speaking_rate_bucket( + config: ServerArgs, + *, + speaking_rate_bucket: int | None = None, + speaking_rate: float | None = None, + speed: float | None = None, + speaking_rate_enabled: bool = False, +) -> int | None: + if not speaking_rate_enabled: + return None + + supplied = [ + speaking_rate_bucket is not None, + speaking_rate is not None, + speed is not None, + ] + if sum(supplied) == 0: + return None + if sum(supplied) > 1: + raise ValueError("Provide only one of speaking_rate_bucket, speaking_rate, or speed.") + + num_buckets = _model_speaking_rate_num_buckets(config) + if num_buckets <= 0: + if speed is not None and speaking_rate_bucket is None and speaking_rate is None: + return None + raise ValueError("Current model does not support speaking-rate conditioning.") + + if speaking_rate_bucket is not None: + bucket = int(speaking_rate_bucket) + if bucket < 0 or bucket >= num_buckets: + raise ValueError( + f"speaking_rate_bucket must be in [0, {num_buckets - 1}], got {bucket}." + ) + return bucket + + ranges = _speaking_rate_bucket_ranges(config) + if ranges and len(ranges) != num_buckets: + raise ValueError( + f"Model has {num_buckets} speaking-rate buckets, but config defines {len(ranges)} ranges." + ) + + if speaking_rate is not None: + return _speaking_rate_bucket_for_rate( + float(speaking_rate), + num_buckets=num_buckets, + ranges=ranges, + ) + + assert speed is not None + speed_value = float(speed) + if speed_value <= 0: + raise ValueError("speed must be positive.") + return _speaking_rate_bucket_for_rate( + _neutral_speaking_rate_bytes_per_second(ranges) * speed_value, + num_buckets=num_buckets, + ranges=ranges, + ) + + +def _normalize_tts_request_language(language: str) -> str: + from zonos2.tokenizer.textnorm import SERVER_TO_NEMO_LANG + + normalized = str(language or "").strip().lower().replace("-", "_") + if normalized not in SERVER_TO_NEMO_LANG: + supported = ", ".join(SERVER_TO_NEMO_LANG) + raise ValueError( + f"Unsupported language code: {language!r}. Supported: {supported}." + ) + return normalized + + +def _model_speaker_background_token_enabled(config: ServerArgs) -> bool: + return bool( + getattr(config.model_config, "speaker_background_token_enabled", False) + or getattr(config, "tts_speaker_background_token_enabled", False) + ) + + +def _model_accurate_mode_token_enabled(config: ServerArgs) -> bool: + return _model_speaker_background_token_enabled(config) and bool( + getattr(config.model_config, "accurate_mode_token_enabled", False) + or getattr(config, "tts_accurate_mode_token_enabled", False) + ) + + +def _model_quality_features(config: ServerArgs) -> list[str]: + raw = getattr(config.model_config, "quality_features", None) + if not raw: + model_buckets = getattr(config.model_config, "quality_buckets", None) or {} + raw = model_buckets.keys() + if not raw: + raw = getattr(config, "tts_quality_features", ()) + if not raw: + server_buckets = getattr(config, "tts_quality_buckets", {}) or {} + raw = server_buckets.keys() + if raw is None: + raw = _QUALITY_METRIC_FIELDS + if isinstance(raw, Mapping) or hasattr(raw, "items"): + return [str(feature) for feature, enabled in raw.items() if bool(enabled)] + if isinstance(raw, str): + return [raw] + return [str(item) for item in (raw or ())] + + +def _model_quality_buckets(config: ServerArgs) -> dict[str, list[str]]: + raw = getattr(config.model_config, "quality_buckets", None) + if not raw: + raw = getattr(config, "tts_quality_buckets", {}) + features = _model_quality_features(config) + return { + feature: [str(item) for item in ((raw or {}).get(feature, ()) or ())] + for feature in features + } + + +def _model_quality_bucket_counts(config: ServerArgs) -> list[int]: + buckets = _model_quality_buckets(config) + return [len(buckets.get(feature, ())) for feature in _model_quality_features(config)] + + +def _model_quality_num_buckets(config: ServerArgs) -> int: + model_value = int(getattr(config.model_config, "quality_num_buckets", 0) or 0) + server_value = int(getattr(config, "tts_quality_num_buckets", 0) or 0) + return model_value or server_value or sum(_model_quality_bucket_counts(config)) + + +def _model_quality_dropout(config: ServerArgs) -> float | dict[str, float] | None: + raw = getattr(config.model_config, "quality_dropout", None) + if raw is None: + raw = getattr(config, "tts_quality_dropout", None) + if raw is None: + return None + if isinstance(raw, Mapping) or hasattr(raw, "items"): + return {str(feature): float(dropout) for feature, dropout in raw.items()} + return float(raw) + + +def _model_quality_dropout_by_feature(config: ServerArgs) -> dict[str, float]: + raw = _model_quality_dropout(config) + features = _model_quality_features(config) + if raw is None: + return dict.fromkeys(features, 0.0) + if isinstance(raw, dict): + default = float(raw.get("default", 0.0)) + return {feature: float(raw.get(feature, default)) for feature in features} + return {feature: float(raw) for feature in features} + + +def _parse_quality_bucket(spec: str) -> tuple[str, float, float | None]: + value = str(spec) + exact = _QUALITY_EXACT_BUCKET_RE.match(value) + if exact is not None: + return "exact", float(exact.group(1)), None + + closed = _QUALITY_CLOSED_BUCKET_RE.match(value) + if closed is not None: + return "range", float(closed.group(1)), float(closed.group(2)) + + open_ended = _QUALITY_OPEN_BUCKET_RE.match(value) + if open_ended is not None: + return "range", float(open_ended.group(1)), None + + raise ValueError( + f"Invalid quality bucket {spec!r}; expected exact values like '0', " + "ranges like '-30--25', or open-ended ranges like '22050+'." + ) + + +def _quality_bucket_specs( + config: ServerArgs, feature: str +) -> list[tuple[str, float, float | None]]: + raw = _model_quality_buckets(config).get(feature, ()) + specs = [_parse_quality_bucket(spec) for spec in raw] + for idx, (kind, low, high) in enumerate(specs): + if not math.isfinite(low): + raise ValueError(f"quality_buckets.{feature} must use finite bucket values.") + if kind == "range": + if high is not None and not math.isfinite(high): + raise ValueError(f"quality_buckets.{feature} must use finite bucket values.") + if high is not None and high <= low: + raise ValueError( + f"quality_buckets.{feature} has an empty or inverted range at index {idx}." + ) + return specs + + +def _quality_bucket_for_value(value: float, config: ServerArgs, feature: str) -> int | None: + try: + value = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(value): + return None + + specs = _quality_bucket_specs(config, feature) + if not specs: + return None + + for idx, (kind, low, _) in enumerate(specs): + if kind == "exact" and math.isclose(value, low, rel_tol=1e-12, abs_tol=1e-9): + return idx + + range_indexes = [idx for idx, (kind, _, _) in enumerate(specs) if kind == "range"] + if not range_indexes: + return None + + for idx in range_indexes: + _, low, high = specs[idx] + if high is None: + if value >= low: + return idx + elif idx == range_indexes[-1]: + if low <= value <= high: + return idx + elif low <= value < high: + return idx + + _, first_low, _ = specs[range_indexes[0]] + if value < first_low: + return range_indexes[0] + return range_indexes[-1] + + +def _quality_control_to_feature_list(value: Any, features: list[str]) -> list[Any]: + if value is None: + return [None] * len(features) + if isinstance(value, dict): + return [value.get(feature) for feature in features] + if isinstance(value, (list, tuple)): + return [value[idx] if idx < len(value) else None for idx in range(len(features))] + raise ValueError("quality_buckets and quality_values must be a list or feature-name object.") + + +def _resolve_quality_buckets( + config: ServerArgs, + *, + quality_buckets: Any = None, + quality_values: Any = None, + quality_enabled: bool = False, +) -> list[int | None] | None: + if not quality_enabled: + return None + + if quality_buckets is None and quality_values is None: + return None + if quality_buckets is not None and quality_values is not None: + raise ValueError("Provide only one of quality_buckets or quality_values.") + + features = _model_quality_features(config) + counts = _model_quality_bucket_counts(config) + if not features or _model_quality_num_buckets(config) <= 0 or sum(counts) <= 0: + raise ValueError("Current model does not support quality conditioning.") + + if any(count <= 0 for count in counts): + raise ValueError("Every configured quality feature must define at least one bucket.") + + if quality_buckets is not None: + raw_buckets = _quality_control_to_feature_list(quality_buckets, features) + resolved: list[int | None] = [] + for feature, count, raw_bucket in zip(features, counts, raw_buckets, strict=True): + if raw_bucket is None: + resolved.append(None) + continue + bucket = int(raw_bucket) + if bucket < 0 or bucket >= count: + raise ValueError( + f"quality_buckets.{feature} must be in [0, {count - 1}], got {bucket}." + ) + resolved.append(bucket) + return resolved + + raw_values = _quality_control_to_feature_list(quality_values, features) + return [ + _quality_bucket_for_value(raw_value, config, feature) if raw_value is not None else None + for feature, raw_value in zip(features, raw_values, strict=True) + ] + + +def _default_quality_buckets(config: ServerArgs) -> list[int | None] | None: + """Resolve the default quality conditioning for models that support it.""" + if _model_quality_num_buckets(config) <= 0: + return None + try: + return _resolve_quality_buckets( + config, quality_buckets=dict(_DEFAULT_QUALITY_BUCKETS), quality_enabled=True + ) + except ValueError: + return None + + +def _apply_fade_out_pcm( + audio_bytes: bytes, fade_out_ms: float, sample_rate: int = 44100 +) -> bytes: + """Apply a cosine fade-out to the tail of float32 PCM bytes.""" + if fade_out_ms <= 0 or not audio_bytes: + return audio_bytes + samples = np.frombuffer(audio_bytes, dtype=np.float32).copy() + n = min(len(samples), int(sample_rate * fade_out_ms / 1000.0)) + if n <= 0: + return audio_bytes + fade = 0.5 * (1.0 + np.cos(np.linspace(0.0, np.pi, n, dtype=np.float32))) + samples[-n:] *= fade + return samples.tobytes() + + +def _normalize_session_id(session_id: str | None) -> str | None: + if session_id is None: + return None + normalized = session_id.strip() + return normalized or None + + +def _require_session_id(session_id: str | None) -> str: + normalized = _normalize_session_id(session_id) + if normalized is None: + raise ValueError("Cached speaker operations require the X-TTS-Session-ID header.") + return normalized + + +def _default_speaker_label( + explicit_label: str | None, + file_name: str | None, + *, + fallback: str, +) -> str: + if explicit_label and explicit_label.strip(): + return explicit_label.strip() + if file_name: + stem = Path(file_name).stem.strip() + if stem: + return stem + return fallback + + +def _slerp_embeddings(v0: torch.Tensor, v1: torch.Tensor, t: float) -> torch.Tensor: + v0 = v0.detach().to(dtype=torch.float32, device="cpu").contiguous().view(-1) + v1 = v1.detach().to(dtype=torch.float32, device="cpu").contiguous().view(-1) + if v0.shape != v1.shape: + raise ValueError( + f"Cannot blend speaker embeddings with different shapes {tuple(v0.shape)} and {tuple(v1.shape)}." + ) + + blend = float(t) + if blend < 0.0 or blend > 1.0: + raise ValueError("speaker_blend_t must be between 0.0 and 1.0.") + + v0_norm = torch.linalg.vector_norm(v0) + 1e-8 + v1_norm = torch.linalg.vector_norm(v1) + 1e-8 + v0_n = v0 / v0_norm + v1_n = v1 / v1_norm + omega = torch.acos(torch.clamp(torch.dot(v0_n, v1_n), -1.0, 1.0)) + if float(omega) < 1e-6: + return ((1.0 - blend) * v0 + blend * v1).contiguous() + + sin_omega = torch.sin(omega) + mixed = ( + (torch.sin((1.0 - blend) * omega) / sin_omega) * v0 + + (torch.sin(blend * omega) / sin_omega) * v1 + ) + return mixed.to(dtype=torch.float32, device="cpu").contiguous() + + +def _serialize_cached_speaker(entry: CachedSpeakerReference) -> dict: + return { + "id": entry.speaker_id, + "label": entry.label, + "source_type": entry.source_type, + "original_name": entry.original_name, + "dimension": int(entry.embedding.numel()), + "created_at": entry.created_at, + "has_preview": bool(entry.audio_bytes), + } + + +def _serialize_default_speaker(entry: DefaultSpeakerReference, expected_dim: int) -> dict: + return { + "id": entry.speaker_id, + "label": entry.label, + "source_type": entry.source_type, + "scope": "default", + "is_default": True, + "original_name": entry.original_name, + "dimension": expected_dim, + "created_at": entry.mtime, + "has_preview": entry.source_type == "audio", + } + + +def _default_voice_root(config: ServerArgs) -> Path | None: + raw = getattr(config, "tts_default_voices_dir", None) + if not raw: + return None + return Path(str(raw)).expanduser() + + +def _default_voice_source_type(path: Path) -> Literal["audio", "embedding_file"] | None: + suffix = path.suffix.lower() + if suffix in _DEFAULT_VOICE_EMBEDDING_EXTENSIONS: + return "embedding_file" + if suffix in _DEFAULT_VOICE_AUDIO_EXTENSIONS: + return "audio" + return None + + +def _default_voice_id(root: Path, path: Path) -> str: + rel = path.relative_to(root).as_posix() + digest = hashlib.sha1(rel.encode("utf-8")).hexdigest()[:16] + return f"default_{digest}" + + +def _default_voice_label(path: Path) -> str: + label = path.stem.replace("_", " ").replace("-", " ").strip() + return label or path.name + + +async def _list_default_speakers(config: ServerArgs) -> list[DefaultSpeakerReference]: + root = _default_voice_root(config) + if root is None: + return [] + if not root.exists() or not root.is_dir(): + logger.warning("TTS default voices directory does not exist or is not a directory: %s", root) + return [] + + root = root.resolve() + expected_dim = _model_speaker_dim(config) + cache_key = (str(root), expected_dim) + + async with _DEFAULT_SPEAKER_CACHE_LOCK: + previous = _DEFAULT_SPEAKER_CACHE.get(cache_key, {}) + current: dict[str, DefaultSpeakerReference] = {} + for path in sorted((p for p in root.rglob("*") if p.is_file()), key=lambda p: p.as_posix()): + source_type = _default_voice_source_type(path) + if source_type is None: + continue + + resolved_path = path.resolve() + try: + stat = resolved_path.stat() + except OSError: + continue + + speaker_id = _default_voice_id(root, path) + existing = previous.get(speaker_id) + if ( + existing is not None + and existing.path == resolved_path + and existing.mtime == stat.st_mtime + and existing.source_type == source_type + ): + current[speaker_id] = existing + continue + + current[speaker_id] = DefaultSpeakerReference( + speaker_id=speaker_id, + label=_default_voice_label(resolved_path), + source_type=source_type, + path=resolved_path, + mtime=stat.st_mtime, + original_name=resolved_path.name, + ) + + _DEFAULT_SPEAKER_CACHE[cache_key] = current + return sorted(current.values(), key=lambda entry: entry.label.lower()) + + +async def _get_default_speaker( + config: ServerArgs, + speaker_id: str, +) -> DefaultSpeakerReference | None: + speakers = await _list_default_speakers(config) + return next((entry for entry in speakers if entry.speaker_id == speaker_id), None) + + +def _read_default_voice_file(entry: DefaultSpeakerReference) -> bytes: + try: + return entry.path.read_bytes() + except OSError as exc: + raise ValueError(f"Failed to read default voice '{entry.label}'.") from exc + + +async def _default_speaker_preview(entry: DefaultSpeakerReference) -> tuple[bytes, str]: + if entry.source_type != "audio": + raise ValueError("Default embedding speakers do not include preview audio.") + if entry.audio_bytes is None: + entry.audio_bytes = _transcode_audio_bytes_to_wav(_read_default_voice_file(entry)) + entry.audio_media_type = "audio/wav" + return entry.audio_bytes, entry.audio_media_type or "audio/wav" + + +async def _load_default_speaker_embedding( + config: ServerArgs, + entry: DefaultSpeakerReference, +) -> torch.Tensor: + expected_dim = _model_speaker_dim(config) + if entry.embedding is None: + if entry.source_type == "embedding_file": + entry.embedding = _load_embedding_vector( + _read_default_voice_file(entry), + expected_dim=expected_dim, + file_name=entry.original_name, + ) + else: + wav_bytes, _ = await _default_speaker_preview(entry) + wav, sample_rate = _decode_wav_bytes(wav_bytes) + entry.embedding = await _compute_speaker_embedding_from_waveform( + wav, + sample_rate, + expected_dim=expected_dim, + ) + + return entry.embedding.detach().to(dtype=torch.float32, device="cpu").contiguous().clone() + + +async def _cache_speaker_reference( + *, + session_id: str, + label: str, + source_type: Literal["audio", "embedding_file"], + embedding: torch.Tensor, + original_name: str | None = None, + audio_bytes: bytes | None = None, + audio_media_type: str | None = None, +) -> CachedSpeakerReference: + entry = CachedSpeakerReference( + speaker_id=f"spk_{uuid.uuid4().hex[:12]}", + label=label, + source_type=source_type, + embedding=embedding.detach().to(dtype=torch.float32, device="cpu").contiguous().clone(), + created_at=time.time(), + original_name=original_name, + audio_bytes=audio_bytes, + audio_media_type=audio_media_type, + ) + async with _SESSION_SPEAKER_CACHE_LOCK: + session_cache = _SESSION_SPEAKER_CACHE.setdefault(session_id, {}) + session_cache[entry.speaker_id] = entry + return entry + + +async def _list_cached_speakers(session_id: str) -> list[CachedSpeakerReference]: + async with _SESSION_SPEAKER_CACHE_LOCK: + session_cache = _SESSION_SPEAKER_CACHE.get(session_id, {}) + return sorted( + session_cache.values(), + key=lambda item: item.created_at, + reverse=True, + ) + + +async def _get_cached_speaker(session_id: str, speaker_id: str) -> CachedSpeakerReference: + async with _SESSION_SPEAKER_CACHE_LOCK: + session_cache = _SESSION_SPEAKER_CACHE.get(session_id, {}) + entry = session_cache.get(speaker_id) + if entry is None: + raise ValueError(f"Unknown cached speaker id '{speaker_id}'.") + return entry + + +def _decode_base64_blob(data: str, field_name: str) -> bytes: + payload = data.strip() + if payload.startswith("data:"): + if "," not in payload: + raise ValueError(f"Invalid data URL for {field_name}.") + payload = payload.split(",", 1)[1] + try: + return base64.b64decode(payload, validate=True) + except Exception as exc: # pragma: no cover - branch depends on invalid user payload + raise ValueError(f"{field_name} must be valid base64-encoded data.") from exc + + +def _decode_wav_bytes(wav_bytes: bytes) -> tuple[torch.Tensor, int]: + try: + with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file: + sample_rate = wav_file.getframerate() + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + n_frames = wav_file.getnframes() + pcm = wav_file.readframes(n_frames) + except Exception as exc: # pragma: no cover - branch depends on invalid user payload + raise ValueError("Reference audio must be a valid PCM WAV file.") from exc + + if len(pcm) == 0: + raise ValueError("Reference audio is empty.") + + pcm_view = memoryview(bytearray(pcm)) + + if sample_width == 1: + audio = torch.frombuffer(pcm_view, dtype=torch.uint8).to(torch.float32) + audio = (audio - 128.0) / 128.0 + elif sample_width == 2: + audio = torch.frombuffer(pcm_view, dtype=torch.int16).to(torch.float32) + audio = audio / 32768.0 + elif sample_width == 4: + audio = torch.frombuffer(pcm_view, dtype=torch.int32).to(torch.float32) + audio = audio / 2147483648.0 + else: + raise ValueError("Unsupported WAV bit depth. Use 8/16/32-bit PCM WAV.") + + if channels > 1: + audio = audio.view(-1, channels).transpose(0, 1).contiguous() + else: + audio = audio.view(1, -1) + + return audio, sample_rate + + +def _transcode_audio_bytes_to_wav(audio_bytes: bytes) -> bytes: + if not audio_bytes: + raise ValueError("Reference file is empty.") + + try: + proc = subprocess.run( + [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-i", + "pipe:0", + "-map", + "0:a:0", + "-vn", + "-ac", + "1", + "-c:a", + "pcm_s16le", + "-f", + "wav", + "pipe:1", + ], + input=audio_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + except FileNotFoundError as exc: + raise ValueError( + "Reference file must contain an audio stream supported by ffmpeg." + ) from exc + + if proc.returncode != 0 or not proc.stdout: + stderr = proc.stderr.decode("utf-8", errors="replace").strip() + suffix = f" ffmpeg said: {stderr}" if stderr else "" + raise ValueError( + "Reference file must contain an audio stream supported by ffmpeg." + + suffix + ) + + logger.debug("Converted reference media to PCM WAV via ffmpeg for speaker embedding.") + return proc.stdout + + +def _decode_audio_bytes(audio_bytes: bytes) -> tuple[torch.Tensor, int]: + return _decode_wav_bytes(_transcode_audio_bytes_to_wav(audio_bytes)) + + +def _load_embedding_vector( + embedding_bytes: bytes, + *, + expected_dim: int, + file_name: str | None = None, +) -> torch.Tensor: + try: + loaded = np.load(io.BytesIO(embedding_bytes), allow_pickle=False) + except Exception as exc: + file_desc = file_name or "Speaker embedding file" + raise ValueError( + f"{file_desc} must be a valid .npy or .npz file saved without pickle." + ) from exc + + arr = None + if isinstance(loaded, np.lib.npyio.NpzFile): + try: + if "emb" in loaded.files: + arr = loaded["emb"] + elif len(loaded.files) == 1: + arr = loaded[loaded.files[0]] + else: + raise ValueError( + "Embedding archive must contain an 'emb' array or exactly one array." + ) + finally: + loaded.close() + else: + arr = loaded + + if isinstance(arr, np.ndarray) and arr.dtype.names: + if "emb" in arr.dtype.names: + arr = arr["emb"] + elif len(arr.dtype.names) == 1: + arr = arr[arr.dtype.names[0]] + else: + raise ValueError("Structured embedding arrays must contain an 'emb' field.") + + arr = np.asarray(arr, dtype=np.float32) + arr = np.squeeze(arr) + + if arr.ndim == 1: + vector = arr + elif arr.ndim == 2: + if arr.shape[0] == 0: + raise ValueError("Speaker embedding file is empty.") + # Multi-row embedding files come from segmented exports; average them into one speaker vector. + vector = arr[0] if arr.shape[0] == 1 else arr.mean(axis=0) + else: + raise ValueError( + f"Unsupported embedding array shape {tuple(arr.shape)}. Expected (D,) or (N, D)." + ) + + vector = np.ascontiguousarray(vector, dtype=np.float32) + if vector.shape[-1] != expected_dim: + raise ValueError( + f"Reference embedding dimension mismatch. Model expects {expected_dim}, " + f"but loaded file has {vector.shape[-1]}." + ) + + return torch.from_numpy(vector).to(dtype=torch.float32, device="cpu") + + +async def _get_speaker_embedder(expected_dim: int): + global _SPEAKER_EMBEDDERS + if expected_dim != 2048: + raise RuntimeError( + f"The release speaker encoder only supports 2048D embeddings; model expects {expected_dim}D." + ) + backend_key = "qwen3" + if backend_key in _SPEAKER_EMBEDDERS: + return _SPEAKER_EMBEDDERS[backend_key] + + async with _SPEAKER_EMBEDDER_LOCK: + if backend_key in _SPEAKER_EMBEDDERS: + return _SPEAKER_EMBEDDERS[backend_key] + try: + from zonos2.models.speaker_cloning import Qwen3SpeakerEmbedding + + embedder = Qwen3SpeakerEmbedding(device=_SPEAKER_EMBEDDER_DEVICE) + except Exception as exc: # pragma: no cover - depends on environment deps + raise RuntimeError( + "Speaker embedding backend failed to load. Ensure torchaudio, " + "transformers, and huggingface_hub are installed and that the " + f"Qwen3 speaker model is accessible. Original error: {type(exc).__name__}: {exc}" + ) from exc + _SPEAKER_EMBEDDERS[backend_key] = embedder + return _SPEAKER_EMBEDDERS[backend_key] + + +async def _compute_speaker_embedding_from_audio( + speaker_audio_base64: str, + expected_dim: int, +) -> torch.Tensor: + audio_bytes = _decode_base64_blob(speaker_audio_base64, "speaker_audio_base64") + wav, sample_rate = _decode_audio_bytes(audio_bytes) + return await _compute_speaker_embedding_from_waveform( + wav, + sample_rate, + expected_dim=expected_dim, + ) + + +async def _compute_speaker_embedding_from_waveform( + wav: torch.Tensor, + sample_rate: int, + *, + expected_dim: int, +) -> torch.Tensor: + embedder = await _get_speaker_embedder(expected_dim) + with torch.inference_mode(): + output = embedder(wav, sample_rate) + + if isinstance(output, tuple): + candidates = [ + tensor.squeeze(0).to(dtype=torch.float32, device="cpu") + for tensor in output + ] + else: + candidates = [output.squeeze(0).to(dtype=torch.float32, device="cpu")] + + for candidate in candidates: + if candidate.numel() == expected_dim: + return candidate + + raise ValueError( + f"Reference embedding dimension mismatch. Model expects {expected_dim}, " + f"but speaker encoder produced {', '.join(str(c.numel()) for c in candidates)}." + ) + + +def _compute_speaker_embedding_from_file( + speaker_embedding_base64: str, + expected_dim: int, + speaker_embedding_name: str | None = None, +) -> torch.Tensor: + embedding_bytes = _decode_base64_blob( + speaker_embedding_base64, + "speaker_embedding_base64", + ) + return _load_embedding_vector( + embedding_bytes, + expected_dim=expected_dim, + file_name=speaker_embedding_name, + ) + + +async def _resolve_cached_speaker_embedding( + *, + config: ServerArgs, + session_id: str | None, + speaker_id: str, +) -> torch.Tensor | None: + if not _model_supports_speaker(config): + logger.info( + "Cached speaker reference provided, but current model does not support speaker embeddings; ignoring." + ) + return None + + default_entry = await _get_default_speaker(config, speaker_id) + if default_entry is not None: + return await _load_default_speaker_embedding(config, default_entry) + + resolved_session_id = _require_session_id(session_id) + entry = await _get_cached_speaker(resolved_session_id, speaker_id) + expected_dim = _model_speaker_dim(config) + actual_dim = int(entry.embedding.numel()) + if actual_dim != expected_dim: + raise ValueError( + f"Cached speaker '{entry.label}' has {actual_dim} dimensions, but the current model expects {expected_dim}." + ) + return entry.embedding.detach().to(dtype=torch.float32, device="cpu").contiguous().clone() + + +async def _resolve_speaker_embedding( + *, + config: ServerArgs, + session_id: str | None = None, + speaker_audio_base64: str | None = None, + speaker_audio_name: str | None = None, + speaker_embedding_base64: str | None = None, + speaker_embedding_name: str | None = None, + speaker_embedding_id: str | None = None, + speaker_blend_embedding_id_a: str | None = None, + speaker_blend_embedding_id_b: str | None = None, + speaker_blend_t: float | None = None, + legacy_speaker_wav_base64: str | None = None, +) -> torch.Tensor | None: + speaker_audio_base64 = speaker_audio_base64 or legacy_speaker_wav_base64 + direct_source_count = sum( + bool(value) for value in (speaker_audio_base64, speaker_embedding_base64) + ) + if direct_source_count > 1: + raise ValueError( + "Provide either a reference audio file or a saved embedding file, not both." + ) + + has_direct_source = direct_source_count == 1 + has_cached_source = bool(speaker_embedding_id) + has_blended_source = bool(speaker_blend_embedding_id_a or speaker_blend_embedding_id_b) + selected_modes = sum(bool(mode) for mode in (has_direct_source, has_cached_source, has_blended_source)) + if selected_modes == 0: + return None + if selected_modes > 1: + raise ValueError( + "Provide speaker conditioning via upload, one cached speaker id, or two cached ids for blending." + ) + + if has_blended_source: + if not speaker_blend_embedding_id_a or not speaker_blend_embedding_id_b: + raise ValueError("Provide both cached speaker ids to blend between two speakers.") + speaker_a = await _resolve_cached_speaker_embedding( + config=config, + session_id=session_id, + speaker_id=speaker_blend_embedding_id_a, + ) + speaker_b = await _resolve_cached_speaker_embedding( + config=config, + session_id=session_id, + speaker_id=speaker_blend_embedding_id_b, + ) + if speaker_a is None or speaker_b is None: + return None + return _slerp_embeddings( + speaker_a, + speaker_b, + 0.5 if speaker_blend_t is None else speaker_blend_t, + ) + + if has_cached_source: + assert speaker_embedding_id is not None + return await _resolve_cached_speaker_embedding( + config=config, + session_id=session_id, + speaker_id=speaker_embedding_id, + ) + + if not _model_supports_speaker(config): + logger.info( + "Speaker reference provided, but current model does not support speaker embeddings; ignoring." + ) + return None + + expected_dim = _model_speaker_dim(config) + if speaker_embedding_base64: + return _compute_speaker_embedding_from_file( + speaker_embedding_base64, + expected_dim=expected_dim, + speaker_embedding_name=speaker_embedding_name, + ) + assert speaker_audio_base64 is not None + return await _compute_speaker_embedding_from_audio( + speaker_audio_base64, + expected_dim=expected_dim, + ) + + +class ModelCard(BaseModel): + id: str + object: str = "model" + created: int = Field(default_factory=lambda: int(time.time())) + owned_by: str = "mini-sglang" + root: str + + +class ModelList(BaseModel): + object: str = "list" + data: List[ModelCard] = Field(default_factory=list) + + +# TTS Request Models +class TTSGenerateRequest(BaseModel): + """Request model for TTS generation.""" + + text: str + language: str = "en_us" + text_normalization: bool = True + temperature: float = 1.15 + topk: int = 106 + top_p: float = 0.0 + min_p: float = 0.18 + max_tokens: int | None = None + fade_out_ms: float = 0.0 + repetition_window: int = 50 + repetition_penalty: float = 1.2 + repetition_codebooks: int = 8 + seed: int | None = None + speaking_rate_enabled: bool = False + speed: float | None = None + speaking_rate: float | None = None + speaking_rate_bucket: int | None = None + quality_enabled: bool = True + # Per-feature quality bucket indices (dict keyed by feature or list in + # feature order). quality_values takes raw metric values instead. + quality_buckets: Dict[str, int | None] | List[int | None] | None = Field( + default_factory=lambda: dict(_DEFAULT_QUALITY_BUCKETS) + ) + quality_values: Dict[str, float | None] | List[float | None] | None = None + # Mark the speaker embedding as having a clean background. + clean_speaker_background: bool = False + # Accurate mode (on) vs expressive mode (off). + accurate_mode: bool = True + stream: bool = True + speaker_audio_base64: str | None = None + speaker_audio_name: str | None = None + speaker_embedding_base64: str | None = None + speaker_embedding_name: str | None = None + speaker_embedding_id: str | None = None + speaker_blend_embedding_id_a: str | None = None + speaker_blend_embedding_id_b: str | None = None + speaker_blend_t: float | None = None + speaker_wav_base64: str | None = None + + +class OpenAISpeechRequest(BaseModel): + """OpenAI-compatible TTS request.""" + + model: str + input: str + voice: str = "alloy" + response_format: str = "pcm" + speaking_rate_enabled: bool = False + speed: float = 1.0 + speaking_rate: float | None = None + speaking_rate_bucket: int | None = None + repetition_window: int = 0 + repetition_penalty: float = 1.0 + repetition_codebooks: int = -1 + speaker_audio_base64: str | None = None + speaker_audio_name: str | None = None + speaker_embedding_base64: str | None = None + speaker_embedding_name: str | None = None + speaker_embedding_id: str | None = None + speaker_blend_embedding_id_a: str | None = None + speaker_blend_embedding_id_b: str | None = None + speaker_blend_t: float | None = None + speaker_wav_base64: str | None = None + + +class TTSSpeakerCacheRequest(BaseModel): + label: str | None = None + speaker_audio_base64: str | None = None + speaker_audio_name: str | None = None + speaker_embedding_base64: str | None = None + speaker_embedding_name: str | None = None + speaker_wav_base64: str | None = None + + +@dataclass +class FrontendManager: + config: ServerArgs + send_tokenizer: ZmqAsyncPushQueue[BaseTokenizerMsg] + recv_tokenizer: ZmqAsyncPullQueue[BaseFrontendMsg] + uid_counter: int = 0 + initialized: bool = False + ack_map: Dict[int, List[TTSAudioReply]] = field(default_factory=dict) + event_map: Dict[int, asyncio.Event] = field(default_factory=dict) + start_time_map: Dict[int, float] = field(default_factory=dict) + + def new_user(self) -> int: + uid = self.uid_counter + self.uid_counter += 1 + self.ack_map[uid] = [] + self.event_map[uid] = asyncio.Event() + self.start_time_map[uid] = time.perf_counter() + return uid + + async def listen(self): + while True: + msg = await self.recv_tokenizer.get() + for msg in _unwrap_msg(msg): + if msg.uid not in self.ack_map: + # Replies can keep arriving after a client abort; drop them. + continue + self.ack_map[msg.uid].append(msg) + self.event_map[msg.uid].set() + + def _create_listener_once(self): + if not self.initialized: + asyncio.create_task(self.listen()) + self.initialized = True + + async def send_one(self, msg: BaseTokenizerMsg): + self._create_listener_once() + await self.send_tokenizer.put(msg) + + async def wait_for_ack(self, uid: int): + event = self.event_map[uid] + + while True: + await event.wait() + event.clear() + + pending = self.ack_map.get(uid) + if pending is None: + # The request was aborted while we were waiting. + return + self.ack_map[uid] = [] + ack = None + for ack in pending: + yield ack + if ack and ack.finished: + break + + self.ack_map.pop(uid, None) + self.event_map.pop(uid, None) + + async def abort_user(self, uid: int): + await asyncio.sleep(0.1) + if uid in self.ack_map: + del self.ack_map[uid] + if uid in self.event_map: + del self.event_map[uid] + self.start_time_map.pop(uid, None) + logger.warning("Aborting request for user %s", uid) + + async def stream_tts_audio(self, uid: int, fade_out_ms: float = 0.0): + """Stream TTS audio chunks as raw PCM bytes. + + When fade_out_ms > 0, the last fade_out_ms of audio is withheld until + the stream finishes so a cosine fade-out can be applied to the tail. + """ + t_start = self.start_time_map.pop(uid, None) + first_byte = True + total_audio_bytes = 0 + chunk_count = 0 + fade_bytes = int(44100 * max(0.0, fade_out_ms) / 1000.0) * 4 + held = b"" + async for ack in self.wait_for_ack(uid): + if isinstance(ack, TTSAudioReply): + if ack.audio_data: + if first_byte and t_start is not None: + logger.info( + "uid=%d TTS TTFB=%.3fms", uid, + (time.perf_counter() - t_start) * 1000, + ) + first_byte = False + total_audio_bytes += len(ack.audio_data) + chunk_count += 1 + if fade_bytes > 0: + held += ack.audio_data + if len(held) > fade_bytes: + yield held[:-fade_bytes] + held = held[-fade_bytes:] + else: + yield ack.audio_data + if ack.finished: + break + if held: + yield _apply_fade_out_pcm(held, fade_out_ms) + if t_start is not None: + e2e_s = time.perf_counter() - t_start + audio_s = total_audio_bytes / (44100 * 4) # float32 @ 44.1kHz + logger.info( + "uid=%d TTS completed: E2E=%.3fms, audio=%.2fs, chunks=%d, RTF=%.2fx", + uid, e2e_s * 1000, audio_s, chunk_count, + audio_s / e2e_s if e2e_s > 0 else 0, + ) + + async def wait_for_tts_audio(self, uid: int, fade_out_ms: float = 0.0): + """Collect all TTS audio data and return as single bytes object.""" + t_start = self.start_time_map.pop(uid, None) + audio_data = b"" + async for ack in self.wait_for_ack(uid): + if isinstance(ack, TTSAudioReply): + if ack.audio_data: + audio_data += ack.audio_data + if ack.finished: + break + audio_data = _apply_fade_out_pcm(audio_data, fade_out_ms) + if t_start is not None: + e2e_s = time.perf_counter() - t_start + audio_s = len(audio_data) / (44100 * 4) + logger.info( + "uid=%d TTS completed (non-stream): E2E=%.3fms, audio=%.2fs, RTF=%.2fx", + uid, e2e_s * 1000, audio_s, + audio_s / e2e_s if e2e_s > 0 else 0, + ) + return audio_data + + def shutdown(self): + self.send_tokenizer.stop() + self.recv_tokenizer.stop() + + +@asynccontextmanager +async def lifespan(_: FastAPI): + yield + # shutdown code here + global _GLOBAL_STATE + if _GLOBAL_STATE is not None: + _GLOBAL_STATE.shutdown() + + +app = FastAPI(title="Zonos2 API Server", version="0.0.1", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/") +async def serve_ui(): + """Serve the TTS web UI.""" + if _UI_HTML.exists(): + return FileResponse( + _UI_HTML, + media_type="text/html", + headers={"Cache-Control": "no-store, max-age=0"}, + ) + return {"error": "tts_ui.html not found"} + + +@app.api_route("/v1", methods=["GET", "POST", "HEAD", "OPTIONS"]) +async def v1_root(): + return {"status": "ok"} + + +@app.get("/v1/models") +async def available_models(): + state = get_global_state() + return ModelList(data=[ModelCard(id=state.config.model_path, root=state.config.model_path)]) + + +# ============================================================================= +# TTS Endpoints +# ============================================================================= + + +@app.post("/v1/audio/speech") +async def create_speech( + req: OpenAISpeechRequest, + x_tts_session_id: str | None = Header(default=None, alias="X-TTS-Session-ID"), +): + """OpenAI-compatible speech endpoint.""" + logger.debug("Received TTS speech request: %s", req.input[:50]) + state = get_global_state() + try: + speaker_embedding = await _resolve_speaker_embedding( + config=state.config, + session_id=x_tts_session_id, + speaker_audio_base64=req.speaker_audio_base64, + speaker_audio_name=req.speaker_audio_name, + speaker_embedding_base64=req.speaker_embedding_base64, + speaker_embedding_name=req.speaker_embedding_name, + speaker_embedding_id=req.speaker_embedding_id, + speaker_blend_embedding_id_a=req.speaker_blend_embedding_id_a, + speaker_blend_embedding_id_b=req.speaker_blend_embedding_id_b, + speaker_blend_t=req.speaker_blend_t, + legacy_speaker_wav_base64=req.speaker_wav_base64, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + try: + speaking_rate_bucket = _resolve_speaking_rate_bucket( + state.config, + speaking_rate_bucket=req.speaking_rate_bucket, + speaking_rate=req.speaking_rate, + speed=req.speed if _field_was_set(req, "speed") else None, + speaking_rate_enabled=req.speaking_rate_enabled, + ) + max_tokens = _model_tts_max_tokens(state.config) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + uid = state.new_user() + + await state.send_one( + TTSTokenizeMsg( + uid=uid, + text=req.input, + sampling_params=TTSSamplingParams( + max_tokens=max_tokens, + repetition_window=req.repetition_window, + repetition_penalty=req.repetition_penalty, + repetition_codebooks=req.repetition_codebooks, + ), + speaker_embedding=speaker_embedding, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=_default_quality_buckets(state.config), + ) + ) + + async def _abort(): + await state.abort_user(uid) + + return StreamingResponse( + state.stream_tts_audio(uid), + media_type="audio/pcm", + headers={ + "X-Audio-Sample-Rate": "44100", + "X-Audio-Channels": "1", + "X-Audio-Format": "float32", + }, + background=BackgroundTask(_abort), + ) + + +@app.post("/tts/generate") +async def tts_generate( + req: TTSGenerateRequest, + x_tts_session_id: str | None = Header(default=None, alias="X-TTS-Session-ID"), +): + """Simple TTS generation endpoint.""" + logger.debug("Received TTS generate request: %s", req.text[:50]) + state = get_global_state() + try: + speaker_embedding = await _resolve_speaker_embedding( + config=state.config, + session_id=x_tts_session_id, + speaker_audio_base64=req.speaker_audio_base64, + speaker_audio_name=req.speaker_audio_name, + speaker_embedding_base64=req.speaker_embedding_base64, + speaker_embedding_name=req.speaker_embedding_name, + speaker_embedding_id=req.speaker_embedding_id, + speaker_blend_embedding_id_a=req.speaker_blend_embedding_id_a, + speaker_blend_embedding_id_b=req.speaker_blend_embedding_id_b, + speaker_blend_t=req.speaker_blend_t, + legacy_speaker_wav_base64=req.speaker_wav_base64, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + try: + language = _normalize_tts_request_language(req.language) + speaking_rate_bucket = _resolve_speaking_rate_bucket( + state.config, + speaking_rate_bucket=req.speaking_rate_bucket, + speaking_rate=req.speaking_rate, + speed=req.speed, + speaking_rate_enabled=req.speaking_rate_enabled, + ) + quality_buckets = _resolve_quality_buckets( + state.config, + quality_buckets=req.quality_buckets, + quality_values=req.quality_values, + quality_enabled=req.quality_enabled and _model_quality_num_buckets(state.config) > 0, + ) + max_tokens = _resolve_tts_max_tokens(state.config, req.max_tokens) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + uid = state.new_user() + + await state.send_one( + TTSTokenizeMsg( + uid=uid, + text=req.text, + language=language, + text_normalization=req.text_normalization, + sampling_params=TTSSamplingParams( + temperature=req.temperature, + topk=req.topk, + top_p=req.top_p, + min_p=req.min_p, + max_tokens=max_tokens, + repetition_window=req.repetition_window, + repetition_penalty=req.repetition_penalty, + repetition_codebooks=req.repetition_codebooks, + seed=req.seed, + ), + speaker_embedding=speaker_embedding, + clean_speaker_background=req.clean_speaker_background, + accurate_mode=req.accurate_mode, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=quality_buckets, + ) + ) + + async def _abort(): + await state.abort_user(uid) + + if req.stream: + return StreamingResponse( + state.stream_tts_audio(uid, fade_out_ms=req.fade_out_ms), + media_type="audio/pcm", + headers={ + "X-Audio-Sample-Rate": "44100", + "X-Audio-Channels": "1", + "X-Audio-Format": "float32", + }, + background=BackgroundTask(_abort), + ) + else: + # Collect full audio then return + audio_data = await state.wait_for_tts_audio(uid, fade_out_ms=req.fade_out_ms) + return Response( + content=audio_data, + media_type="audio/pcm", + headers={ + "X-Audio-Sample-Rate": "44100", + "X-Audio-Channels": "1", + "X-Audio-Format": "float32", + }, + ) + + +@app.post("/tts/speakers") +async def tts_cache_speaker( + req: TTSSpeakerCacheRequest, + x_tts_session_id: str | None = Header(default=None, alias="X-TTS-Session-ID"), +): + """Cache a speaker embedding for later selection in the current session.""" + state = get_global_state() + try: + session_id = _require_session_id(x_tts_session_id) + if not _model_supports_speaker(state.config): + raise ValueError("Current model does not support speaker embeddings.") + + speaker_audio_base64 = req.speaker_audio_base64 or req.speaker_wav_base64 + direct_source_count = sum( + bool(value) + for value in (speaker_audio_base64, req.speaker_embedding_base64) + ) + if direct_source_count == 0: + raise ValueError("Provide an audio file or saved embedding file to cache.") + if direct_source_count > 1: + raise ValueError( + "Provide either a reference audio file or a saved embedding file, not both." + ) + + expected_dim = _model_speaker_dim(state.config) + if speaker_audio_base64: + audio_name = req.speaker_audio_name + audio_bytes = _decode_base64_blob(speaker_audio_base64, "speaker_audio_base64") + preview_wav_bytes = _transcode_audio_bytes_to_wav(audio_bytes) + wav, sample_rate = _decode_wav_bytes(preview_wav_bytes) + embedding = await _compute_speaker_embedding_from_waveform( + wav, + sample_rate, + expected_dim=expected_dim, + ) + entry = await _cache_speaker_reference( + session_id=session_id, + label=_default_speaker_label(req.label, audio_name, fallback="Speaker audio"), + source_type="audio", + embedding=embedding, + original_name=audio_name, + audio_bytes=preview_wav_bytes, + audio_media_type="audio/wav", + ) + else: + embedding_name = req.speaker_embedding_name + embedding = _compute_speaker_embedding_from_file( + req.speaker_embedding_base64, + expected_dim=expected_dim, + speaker_embedding_name=embedding_name, + ) + entry = await _cache_speaker_reference( + session_id=session_id, + label=_default_speaker_label(req.label, embedding_name, fallback="Speaker embedding"), + source_type="embedding_file", + embedding=embedding, + original_name=embedding_name, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + return _serialize_cached_speaker(entry) + + +@app.get("/tts/speakers") +async def tts_list_speakers( + x_tts_session_id: str | None = Header(default=None, alias="X-TTS-Session-ID"), +): + """List default disk speakers and cached speakers for the current session.""" + state = get_global_state() + expected_dim = _model_speaker_dim(state.config) + default_entries = await _list_default_speakers(state.config) + + entries: list[CachedSpeakerReference] = [] + session_id = _normalize_session_id(x_tts_session_id) + if session_id is not None: + entries = await _list_cached_speakers(session_id) + + return { + "speakers": [ + *[_serialize_default_speaker(entry, expected_dim) for entry in default_entries], + *[_serialize_cached_speaker(entry) for entry in entries], + ] + } + + +@app.get("/tts/speakers/{speaker_id}/preview") +async def tts_preview_speaker( + speaker_id: str, + x_tts_session_id: str | None = Header(default=None, alias="X-TTS-Session-ID"), +): + """Return the cached reference audio for a speaker, when available.""" + state = get_global_state() + try: + default_entry = await _get_default_speaker(state.config, speaker_id) + if default_entry is not None: + audio_bytes, media_type = await _default_speaker_preview(default_entry) + return Response(content=audio_bytes, media_type=media_type) + + session_id = _require_session_id(x_tts_session_id) + entry = await _get_cached_speaker(session_id, speaker_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + if not entry.audio_bytes: + raise HTTPException(status_code=404, detail="Cached speaker preview audio is not available.") + + return Response( + content=entry.audio_bytes, + media_type=entry.audio_media_type or "audio/wav", + ) + + +@app.get("/tts/capabilities") +async def tts_capabilities(): + """Return TTS feature flags for the currently loaded model.""" + from zonos2.tokenizer.textnorm import SERVER_TO_NEMO_LANG, normalization_enabled + + state = get_global_state() + speaking_rate_num_buckets = _model_speaking_rate_num_buckets(state.config) + quality_num_buckets = _model_quality_num_buckets(state.config) + return { + "text_normalization_enabled": normalization_enabled(), + "text_norm_languages": list(SERVER_TO_NEMO_LANG), + "speaker_enabled": _model_supports_speaker(state.config), + "speaker_embedding_dim": _model_speaker_dim(state.config), + "n_codebooks": int(getattr(state.config.model_config, "n_codebooks", 9)), + "max_tokens": _model_tts_max_tokens(state.config), + "speaking_rate_enabled": speaking_rate_num_buckets > 0, + "speaking_rate_num_buckets": speaking_rate_num_buckets, + "speaking_rate_buckets": _model_speaking_rate_buckets(state.config), + "quality_enabled": quality_num_buckets > 0, + "quality_num_buckets": quality_num_buckets, + "quality_features": _model_quality_features(state.config), + "quality_buckets": _model_quality_buckets(state.config), + "quality_dropout_by_feature": _model_quality_dropout_by_feature(state.config), + "default_quality_buckets": dict(_DEFAULT_QUALITY_BUCKETS), + "speaker_background_token_enabled": _model_speaker_background_token_enabled( + state.config + ), + "accurate_mode_token_enabled": _model_accurate_mode_token_enabled(state.config), + "speaker_audio_upload": True, + "speaker_embedding_upload": True, + "speaker_embedding_cache": True, + "speaker_embedding_blend": True, + "default_voices_enabled": bool(_default_voice_root(state.config)), + } + + +async def tts_shell(): + """Interactive TTS shell for generating and playing audio.""" + from prompt_toolkit import PromptSession + from prompt_toolkit.completion import WordCompleter + + commands = [ + "/exit", + "/save", + "/play", + "/temperature", + "/topk", + "/repwindow", + "/reppenalty", + "/repcodebooks", + ] + completer = WordCompleter(commands) + session = PromptSession("TTS> ", completer=completer) + + # TTS settings + temperature = 1.15 + topk = 106 + repetition_window = 50 + repetition_penalty = 1.2 + repetition_codebooks = 8 + last_audio: bytes | None = None + + # Try to import sounddevice for audio playback + try: + import numpy as np + import sounddevice as sd + + can_play_audio = True + except ImportError: + can_play_audio = False + print("Note: Install 'sounddevice' to enable audio playback") + + print("TTS Shell - Enter text to generate speech") + print( + "Commands: /exit, /save , /play, /temperature , " + "/topk , /repwindow , /reppenalty , " + "/repcodebooks " + ) + print() + + try: + while True: + cmd = (await session.prompt_async()).strip() + if cmd == "": + continue + + if cmd.startswith("/"): + if cmd == "/exit": + return + elif cmd.startswith("/save"): + parts = cmd.split(maxsplit=1) + if len(parts) < 2: + print("Usage: /save ") + continue + filename = parts[1] + if last_audio is None: + print("No audio to save. Generate some speech first.") + continue + _save_wav(last_audio, filename) + print(f"Saved to {filename}") + continue + elif cmd == "/play": + if not can_play_audio: + print("Audio playback not available. Install 'sounddevice'.") + continue + if last_audio is None: + print("No audio to play. Generate some speech first.") + continue + audio = np.frombuffer(last_audio, dtype=np.float32) + sd.play(audio, samplerate=44100) + sd.wait() + continue + elif cmd.startswith("/temperature"): + parts = cmd.split() + if len(parts) < 2: + print(f"Current temperature: {temperature}") + continue + temperature = float(parts[1]) + print(f"Temperature set to {temperature}") + continue + elif cmd.startswith("/topk"): + parts = cmd.split() + if len(parts) < 2: + print(f"Current topk: {topk}") + continue + topk = int(parts[1]) + print(f"Top-k set to {topk}") + continue + elif cmd.startswith("/repwindow"): + parts = cmd.split() + if len(parts) < 2: + print(f"Current repetition window: {repetition_window}") + continue + repetition_window = max(int(parts[1]), 0) + print(f"Repetition window set to {repetition_window}") + continue + elif cmd.startswith("/reppenalty"): + parts = cmd.split() + if len(parts) < 2: + print(f"Current repetition penalty: {repetition_penalty}") + continue + repetition_penalty = max(float(parts[1]), 1.0) + print(f"Repetition penalty set to {repetition_penalty}") + continue + elif cmd.startswith("/repcodebooks"): + parts = cmd.split() + if len(parts) < 2: + print(f"Current repetition codebooks: {repetition_codebooks}") + continue + repetition_codebooks = int(parts[1]) + print(f"Repetition codebooks set to {repetition_codebooks}") + continue + else: + print(f"Unknown command: {cmd}") + continue + + # Generate TTS + state = get_global_state() + uid = state.new_user() + + await state.send_one( + TTSTokenizeMsg( + uid=uid, + text=cmd, + sampling_params=TTSSamplingParams( + temperature=temperature, + topk=topk, + repetition_window=repetition_window, + repetition_penalty=repetition_penalty, + repetition_codebooks=repetition_codebooks, + ), + ) + ) + + # Collect audio + print("Generating...", end="", flush=True) + audio_chunks = [] + async for ack in state.wait_for_ack(uid): + if isinstance(ack, TTSAudioReply): + if ack.audio_data: + audio_chunks.append(ack.audio_data) + print(".", end="", flush=True) + if ack.finished: + break + + if audio_chunks: + last_audio = b"".join(audio_chunks) + duration = len(last_audio) / (44100 * 4) # float32 = 4 bytes + print(f" Done! ({duration:.2f}s)") + + # Auto-play if sounddevice is available + if can_play_audio: + audio = np.frombuffer(last_audio, dtype=np.float32) + sd.play(audio, samplerate=44100) + sd.wait() + else: + print(" No audio generated") + + except EOFError: + # user pressed Ctrl-D + pass + finally: + print("Exiting TTS shell...") + await asyncio.sleep(0.1) + get_global_state().shutdown() + # then kill all the subprocesses + import psutil + + parent = psutil.Process() + for child in parent.children(recursive=True): + child.kill() + + +def _save_wav(audio_bytes: bytes, path: str, sample_rate: int = 44100) -> None: + """Save PCM audio bytes to a WAV file.""" + import wave + + import numpy as np + + audio = np.frombuffer(audio_bytes, dtype=np.float32) + # Convert to int16 for WAV + audio_int16 = (audio * 32767).astype(np.int16) + + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) # 16-bit + f.setframerate(sample_rate) + f.writeframes(audio_int16.tobytes()) + + +def run_api_server(config: ServerArgs, start_backend: Callable[[], None], run_shell: bool) -> None: + """ + Run the frontend API server (FastAPI + uvicorn) and wire it to the tokenizer process via ZMQ. + + Args: + config: Server configuration (host/port, ZMQ IPC addresses, etc). + start_backend: Callback that launches the backend worker processes (TP schedulers + + tokenizer/detokenizer). + run_shell: If True, run an interactive terminal shell instead of starting uvicorn. + """ + + global _GLOBAL_STATE + + if run_shell: + assert not config.use_dummy_weight, "Shell mode does not support dummy weights." + + host = config.server_host + port = config.server_port + + assert _GLOBAL_STATE is None, "Global state is already initialized" + _GLOBAL_STATE = FrontendManager( + config=config, + recv_tokenizer=ZmqAsyncPullQueue( + config.zmq_frontend_addr, + create=True, + decoder=BaseFrontendMsg.decoder, + ), + send_tokenizer=ZmqAsyncPushQueue( + config.zmq_tokenizer_addr, + create=config.frontend_create_tokenizer_link, + encoder=BaseTokenizerMsg.encoder, + ), + ) + + # start the backend here + start_backend() + + logger.info(f"API server is ready to serve on {host}:{port}") + if not run_shell: + uvicorn.run(app, host=host, port=port) + else: + asyncio.run(tts_shell()) diff --git a/zonos2/server/args.py b/zonos2/server/args.py new file mode 100644 index 0000000000000000000000000000000000000000..1c07b27d63fcbb101067e9843408610a6f6156b1 --- /dev/null +++ b/zonos2/server/args.py @@ -0,0 +1,393 @@ +from __future__ import annotations + +import argparse +import os +from dataclasses import dataclass, field +from typing import Dict, List, Tuple + +import torch +from zonos2.distributed import DistributedInfo +from zonos2.scheduler import SchedulerConfig +from zonos2.utils import cached_load_checkpoint_config, init_logger + + +@dataclass(frozen=True) +class ServerArgs(SchedulerConfig): + server_host: str = "127.0.0.1" + server_port: int = 1919 + num_tokenizer: int = 0 + silent_output: bool = False + + # TTS-specific settings + tts_n_codebooks: int = 9 + tts_audio_pad_id: int = 1025 + tts_text_vocab: int | None = None + tts_sample_rate: int = 44100 + tts_speaker_background_token_enabled: bool = False + tts_accurate_mode_token_enabled: bool = False + tts_speaking_rate_num_buckets: int = 0 + tts_speaking_rate_buckets: Tuple[str, ...] = () + tts_quality_num_buckets: int = 0 + tts_quality_features: Tuple[str, ...] = () + tts_quality_buckets: Dict[str, Tuple[str, ...]] = field(default_factory=dict) + tts_quality_dropout: Dict[str, float] | None = None + tts_default_voices_dir: str | None = None + + @property + def share_tokenizer(self) -> bool: + return self.num_tokenizer == 0 + + @property + def zmq_frontend_addr(self) -> str: + return "ipc:///tmp/zonos2_3" + self._unique_suffix + + @property + def zmq_tokenizer_addr(self) -> str: + if self.share_tokenizer: + return self.zmq_detokenizer_addr + result = "ipc:///tmp/zonos2_4" + self._unique_suffix + assert result != self.zmq_detokenizer_addr + return result + + @property + def tokenizer_create_addr(self) -> bool: + return self.share_tokenizer + + @property + def backend_create_detokenizer_link(self) -> bool: + return not self.share_tokenizer + + @property + def frontend_create_tokenizer_link(self) -> bool: + return not self.share_tokenizer + + @property + def distributed_addr(self) -> str: + return f"tcp://127.0.0.1:{self.server_port + 1}" + + +def parse_args(args: List[str], run_shell: bool = False) -> Tuple[ServerArgs, bool]: + """ + Parse command line arguments and return an EngineConfig. + + Args: + args: Command line arguments (e.g., sys.argv[1:]) + + Returns: + EngineConfig instance with parsed arguments + """ + from zonos2.attention import validate_backend + from zonos2.kvcache import SUPPORTED_CACHE_MANAGER + + def validate_moe_backend(backend: str) -> str: + """Validate MoE backend argument.""" + # Accept any non-empty string as backend name + # Specific backend validation can be added later as backends are implemented + if not backend: + from argparse import ArgumentTypeError + + raise ArgumentTypeError(f"MoE backend must be a non-empty string, got: {backend}") + return backend + + parser = argparse.ArgumentParser(description="Zonos2 Server Arguments") + + parser.add_argument( + "--model-path", + type=str, + required=True, + help="Zonos2 TTS checkpoint: a Hugging Face repo id (e.g. Zyphra/ZONOS2) or a local path.", + ) + + parser.add_argument( + "--dtype", + type=str, + default="auto", + choices=["auto", "float16", "bfloat16", "float32"], + help="Data type for model weights and activations. 'auto' will use FP16 for FP32/FP16 models and BF16 for BF16 models.", + ) + + parser.add_argument( + "--tensor-parallel-size", + "--tp-size", + type=int, + default=1, + help="The tensor parallelism size.", + ) + + parser.add_argument( + "--max-running-requests", + type=int, + dest="max_running_req", + default=ServerArgs.max_running_req, + help="The maximum number of running requests.", + ) + + parser.add_argument( + "--max-seq-len-override", + type=int, + default=ServerArgs.max_seq_len_override, + help="The maximum sequence length override.", + ) + + parser.add_argument( + "--memory-ratio", + type=float, + default=ServerArgs.memory_ratio, + help="The fraction of GPU memory to use for KV cache.", + ) + + assert ServerArgs.use_dummy_weight == False + parser.add_argument( + "--dummy-weight", + action="store_true", + dest="use_dummy_weight", + help="Use dummy weights for testing.", + ) + + assert ServerArgs.use_pynccl == True + parser.add_argument( + "--disable-pynccl", + action="store_false", + dest="use_pynccl", + help="Disable PyNCCL for tensor parallelism.", + ) + + parser.add_argument( + "--host", + type=str, + dest="server_host", + default=ServerArgs.server_host, + help="The host address for the server.", + ) + + parser.add_argument( + "--port", + type=int, + dest="server_port", + default=ServerArgs.server_port, + help="The port number for the server to listen on.", + ) + + parser.add_argument( + "--cuda-graph-max-bs", + "--graph", + type=int, + default=ServerArgs.cuda_graph_max_bs, + help="The maximum batch size for CUDA graph capture. None means auto-tuning based on the GPU memory.", + ) + + parser.add_argument( + "--num-tokenizer", + "--tokenizer-count", + type=int, + default=ServerArgs.num_tokenizer, + help="The number of tokenizer processes to launch. 0 means the tokenizer is shared with the detokenizer.", + ) + + parser.add_argument( + "--max-prefill-length", + "--max-extend-length", + type=int, + dest="max_extend_tokens", + default=ServerArgs.max_extend_tokens, + help="Chunk Prefill maximum chunk size in tokens.", + ) + + parser.add_argument( + "--num-pages", + "--num-tokens", + dest="num_page_override", + type=int, + default=ServerArgs.num_page_override, + help="Set the maximum number of pages for KVCache.", + ) + + parser.add_argument( + "--attention-backend", + "--attn", + type=validate_backend, + default=ServerArgs.attention_backend, + help="The attention backend to use. If two backends are specified," + " the first one is used for prefill and the second one for decode.", + ) + + parser.add_argument( + "--cache-type", + type=str, + default=ServerArgs.cache_type, + choices=SUPPORTED_CACHE_MANAGER.supported_names(), + help="The KV cache management strategy.", + ) + + parser.add_argument( + "--moe-backend", + type=validate_moe_backend, + default=ServerArgs.moe_backend, + help="The MoE backend to use.", + ) + + parser.add_argument( + "--shell-mode", + action="store_true", + help="Run the server in shell mode.", + ) + + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging (shorthand for --log-level DEBUG).", + ) + + parser.add_argument( + "--log-level", + type=str, + default=None, + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + help="Set the log level (default: INFO). Overrides LOG_LEVEL env var.", + ) + + # TTS arguments + parser.add_argument( + "--tts-n-codebooks", + type=int, + default=ServerArgs.tts_n_codebooks, + help="Number of audio codebooks for TTS (default: 9).", + ) + + parser.add_argument( + "--tts-audio-pad-id", + type=int, + default=ServerArgs.tts_audio_pad_id, + help="Audio padding token ID for TTS (default: 1025).", + ) + + parser.add_argument( + "--tts-text-vocab", + type=int, + default=ServerArgs.tts_text_vocab, + help="Text vocabulary size for TTS. If not set, auto-detected from model.", + ) + + parser.add_argument( + "--tts-sample-rate", + type=int, + default=ServerArgs.tts_sample_rate, + help="Audio sample rate for TTS output (default: 44100 for DAC).", + ) + + parser.add_argument( + "--tts-default-voices-dir", + type=str, + default=ServerArgs.tts_default_voices_dir, + help="Directory of default speaker audio or .npy/.npz embedding files to pre-populate in the TTS UI.", + ) + + # Parse arguments + kwargs = parser.parse_args(args).__dict__.copy() + + # Apply log level before any init_logger calls + debug_mode = kwargs.pop("debug") + log_level = kwargs.pop("log_level") + if debug_mode: + from zonos2.utils import set_log_level + + set_log_level("DEBUG") + elif log_level is not None: + from zonos2.utils import set_log_level + + set_log_level(log_level) + + # resolve some arguments + run_shell |= kwargs.pop("shell_mode") + if run_shell: + kwargs["cuda_graph_max_bs"] = 1 + kwargs["max_running_req"] = 1 + kwargs["silent_output"] = True + + if kwargs["model_path"].startswith("~"): + kwargs["model_path"] = os.path.expanduser(kwargs["model_path"]) + if kwargs.get("tts_default_voices_dir"): + kwargs["tts_default_voices_dir"] = os.path.expanduser(kwargs["tts_default_voices_dir"]) + + DTYPE_MAP = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + + checkpoint_config = cached_load_checkpoint_config(kwargs["model_path"]) + + if (dtype_str := kwargs["dtype"]) != "auto": + kwargs["dtype"] = DTYPE_MAP[dtype_str] + else: + dtype_or_str = getattr(checkpoint_config, "dtype", "bfloat16") + if isinstance(dtype_or_str, str): + kwargs["dtype"] = DTYPE_MAP.get(dtype_or_str, torch.bfloat16) + else: + kwargs["dtype"] = dtype_or_str + + kwargs["tp_info"] = DistributedInfo(0, kwargs["tensor_parallel_size"]) + del kwargs["tensor_parallel_size"] + + # Auto-detect TTS settings from model config if not provided + if kwargs.get("tts_text_vocab") is None: + text_vocab = getattr(checkpoint_config, "text_vocab", None) + if text_vocab is not None: + kwargs["tts_text_vocab"] = text_vocab + + # Auto-detect n_codebooks + if kwargs.get("tts_n_codebooks") == ServerArgs.tts_n_codebooks: # still default + n_codebooks = getattr(checkpoint_config, "n_codebooks", None) + if n_codebooks is not None: + kwargs["tts_n_codebooks"] = n_codebooks + + # Auto-detect audio_pad_id + if kwargs.get("tts_audio_pad_id") == ServerArgs.tts_audio_pad_id: # still default + audio_pad_id = getattr(checkpoint_config, "audio_pad_id", None) + if audio_pad_id is not None: + kwargs["tts_audio_pad_id"] = audio_pad_id + + speaking_rate_num_buckets = int(getattr(checkpoint_config, "speaking_rate_num_buckets", 0) or 0) + if speaking_rate_num_buckets > 0: + kwargs["tts_speaking_rate_num_buckets"] = speaking_rate_num_buckets + kwargs["tts_speaking_rate_buckets"] = tuple( + getattr(checkpoint_config, "speaking_rate_buckets", None) or () + ) + + if bool(getattr(checkpoint_config, "speaker_background_token_enabled", False)): + kwargs["tts_speaker_background_token_enabled"] = True + if bool(getattr(checkpoint_config, "accurate_mode_token_enabled", False)): + kwargs["tts_accurate_mode_token_enabled"] = True + + quality_num_buckets = int(getattr(checkpoint_config, "quality_num_buckets", 0) or 0) + if quality_num_buckets > 0: + raw_quality_buckets = getattr(checkpoint_config, "quality_buckets", None) or {} + raw_quality_features = ( + getattr(checkpoint_config, "quality_features", None) or raw_quality_buckets.keys() + ) + quality_features = tuple(str(item) for item in (raw_quality_features or ())) + kwargs["tts_quality_num_buckets"] = quality_num_buckets + kwargs["tts_quality_features"] = quality_features + kwargs["tts_quality_buckets"] = { + str(feature): tuple( + str(item) for item in (raw_quality_buckets.get(feature, ()) or ()) + ) + for feature in quality_features + } + kwargs["tts_quality_dropout"] = getattr(checkpoint_config, "quality_dropout", None) + + # Debug: print TTS settings + logger = init_logger(__name__) + logger.info( + f"TTS settings: n_codebooks={kwargs.get('tts_n_codebooks')}, " + f"text_vocab={kwargs.get('tts_text_vocab')}, " + f"speaking_rate_num_buckets={kwargs.get('tts_speaking_rate_num_buckets', 0)}, " + f"quality_num_buckets={kwargs.get('tts_quality_num_buckets', 0)}, " + f"speaker_background_token={kwargs.get('tts_speaker_background_token_enabled', False)}, " + f"accurate_mode_token={kwargs.get('tts_accurate_mode_token_enabled', False)}" + ) + + result = ServerArgs(**kwargs) + logger = init_logger(__name__) + logger.info(f"Parsed arguments:\n{result}") + return result, run_shell diff --git a/zonos2/server/launch.py b/zonos2/server/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e833ccfa52df53c6e15e950294b5610eb6215a --- /dev/null +++ b/zonos2/server/launch.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import logging +import multiprocessing as mp +import sys +from dataclasses import replace +from typing import TYPE_CHECKING + +from zonos2.distributed import DistributedInfo +from zonos2.utils import init_logger + +if TYPE_CHECKING: + from .args import ServerArgs + + +def _run_scheduler(args: ServerArgs, ack_queue: mp.Queue[str]) -> None: + import torch + + # Multi-GPU tensor parallel workers bind to their rank-specific device + # before importing model code. + if torch.cuda.is_available() and args.tp_info.size > 1: + torch.cuda.set_device(args.tp_info.rank) + + from zonos2.scheduler import TTSScheduler + + with torch.inference_mode(): + scheduler = TTSScheduler(args) + scheduler.sync_all_ranks() + + if args.tp_info.is_primary(): + ack_queue.put("Scheduler is ready") + + if args.silent_output: + logging.disable(logging.INFO) + + try: + scheduler.run_forever() + except KeyboardInterrupt: + logger = init_logger(__name__) + if scheduler.tp_info.is_primary(): + print() # for a clean newline after ^C + logger.info("Scheduler exiting gracefully...") + scheduler.shutdown() + + +def launch_server(run_shell: bool = False) -> None: + from .api_server import run_api_server + from .args import parse_args + + server_args, run_shell = parse_args(sys.argv[1:], run_shell) + logger = init_logger(__name__, "initializer") + + def start_subprocess() -> None: + import multiprocessing as mp + + from zonos2.tokenizer import tokenize_worker + + mp.set_start_method("spawn", force=True) + + world_size = server_args.tp_info.size + # a multiprocessing queue to receive ack from subprocesses + # so that we can guarantee all subprocesses are ready + ack_queue: mp.Queue[str] = mp.Queue() + + for i in range(world_size): + new_args = replace( + server_args, + tp_info=DistributedInfo(i, world_size), + ) + mp.Process( + target=_run_scheduler, + args=(new_args, ack_queue), + daemon=False, + name=f"zonos2-TP{i}-scheduler", + ).start() + + num_tokenizers = server_args.num_tokenizer + + # Common tokenizer kwargs + tokenizer_common_kwargs = { + "backend_addr": server_args.zmq_backend_addr, + "frontend_addr": server_args.zmq_frontend_addr, + "local_bs": 1, + "ack_queue": ack_queue, + "n_codebooks": server_args.tts_n_codebooks, + "audio_pad_id": server_args.tts_audio_pad_id, + "text_vocab": server_args.tts_text_vocab, + "speaking_rate_num_buckets": server_args.tts_speaking_rate_num_buckets, + "quality_bucket_counts": [ + len(server_args.tts_quality_buckets.get(feature, ())) + for feature in server_args.tts_quality_features + ], + "speaker_background_num_buckets": ( + 2 if server_args.tts_speaker_background_token_enabled else 0 + ), + "accurate_mode_num_buckets": ( + 1 + if server_args.tts_speaker_background_token_enabled + and server_args.tts_accurate_mode_token_enabled + else 0 + ), + } + + # DeTokenizer, only 1 + mp.Process( + target=tokenize_worker, + kwargs={ + **tokenizer_common_kwargs, + "addr": server_args.zmq_detokenizer_addr, + "create": server_args.tokenizer_create_addr, + "tokenizer_id": num_tokenizers, + }, + daemon=False, + name="zonos2-detokenizer-0", + ).start() + for i in range(num_tokenizers): + mp.Process( + target=tokenize_worker, + kwargs={ + **tokenizer_common_kwargs, + "addr": server_args.zmq_tokenizer_addr, + "create": server_args.tokenizer_create_addr, + "tokenizer_id": i, + }, + daemon=False, + name=f"zonos2-tokenizer-{i}", + ).start() + + # Wait for acknowledgments from all worker processes: + # - world_size schedulers (but only primary rank sends ack) + # - num_tokenizers tokenizers + # - 1 detokenizer + # Total acks expected: 1 + num_tokenizers + 1 = num_tokenizers + 2 + for _ in range(num_tokenizers + 2): + logger.info(ack_queue.get()) + + run_api_server(server_args, start_subprocess, run_shell=run_shell) + + +if __name__ == "__main__": + launch_server() diff --git a/zonos2/shell.py b/zonos2/shell.py new file mode 100644 index 0000000000000000000000000000000000000000..693000949c0059ee84e9471142ae8ac2060de0ab --- /dev/null +++ b/zonos2/shell.py @@ -0,0 +1,4 @@ +from .server import launch_server + +if __name__ == "__main__": + launch_server(run_shell=True) diff --git a/zonos2/tokenizer/__init__.py b/zonos2/tokenizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..09c04f084cb90dc699c56ac2f59052d570d3fc54 --- /dev/null +++ b/zonos2/tokenizer/__init__.py @@ -0,0 +1,3 @@ +from .server import tokenize_worker + +__all__ = ["tokenize_worker"] diff --git a/zonos2/tokenizer/server.py b/zonos2/tokenizer/server.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d71e355d9d9ebd6bebc248e3709debcf50834a --- /dev/null +++ b/zonos2/tokenizer/server.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import multiprocessing as mp +import threading +from typing import List + +import torch +from zonos2.message import ( + BaseBackendMsg, + BaseFrontendMsg, + BaseTokenizerMsg, + BatchTokenizerMsg, + BatchTTSBackendMsg, + BatchTTSFrontendMsg, + BatchTTSTokenizerMsg, + TTSAudioReply, + TTSDetokenizeMsg, + TTSTokenizeMsg, + TTSUserMsg, +) +from zonos2.tts.prompt import TTSPromptBuilder, TTSPromptConfig +from zonos2.utils import ZmqPullQueue, ZmqPushQueue, init_logger + + +def _unwrap_msg( + msg: BaseTokenizerMsg | BatchTokenizerMsg | BatchTTSTokenizerMsg, +) -> List[BaseTokenizerMsg]: + if isinstance(msg, BatchTokenizerMsg): + return msg.data + if isinstance(msg, BatchTTSTokenizerMsg): + return msg.data + return [msg] + + +@torch.inference_mode() +def tokenize_worker( + *, + addr: str, + create: bool, + backend_addr: str, + frontend_addr: str, + local_bs: int, + tokenizer_id: int = -1, + ack_queue: mp.Queue[str] | None = None, + n_codebooks: int = 9, + audio_pad_id: int = 1025, + text_vocab: int | None = None, + speaking_rate_num_buckets: int = 0, + quality_bucket_counts: List[int] | None = None, + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> None: + send_backend = ZmqPushQueue(backend_addr, create=False, encoder=BaseBackendMsg.encoder) + send_frontend = ZmqPushQueue(frontend_addr, create=False, encoder=BaseFrontendMsg.encoder) + recv_listener = ZmqPullQueue(addr, create=create, decoder=BatchTokenizerMsg.decoder) + assert local_bs > 0 + logger = init_logger(__name__, f"tokenizer_{tokenizer_id}") + + from .textnorm import TTSTextNormalizer, normalization_enabled + from .vocoder import TTSVocoderManager + + if text_vocab is None: + raise ValueError("TTS mode requires text_vocab from the model config.") + textnorm_enabled = normalization_enabled() + text_normalizer = TTSTextNormalizer() if textnorm_enabled else None + if text_normalizer is not None: + # Compile/load the English grammars up front so the first request does + # not pay the one-time WFST construction cost. + threading.Thread( + target=text_normalizer.warmup, args=(["en"],), daemon=True + ).start() + tts_prompt_builder = TTSPromptBuilder( + TTSPromptConfig( + n_codebooks=n_codebooks, + audio_pad_id=audio_pad_id, + text_vocab=text_vocab, + speaking_rate_num_buckets=speaking_rate_num_buckets, + quality_bucket_counts=tuple(quality_bucket_counts or ()), + speaker_background_num_buckets=speaker_background_num_buckets, + accurate_mode_num_buckets=accurate_mode_num_buckets, + prepend_silence=True, + ) + ) + tts_vocoder_manager = TTSVocoderManager( + n_codebooks=n_codebooks, + audio_pad_id=audio_pad_id, + ) + + if ack_queue is not None: + ack_queue.put(f"Tokenize server {tokenizer_id} is ready") + + try: + while True: + pending_msg = _unwrap_msg(recv_listener.get()) + while len(pending_msg) < local_bs and not recv_listener.empty(): + pending_msg.extend(_unwrap_msg(recv_listener.get())) + + logger.debug(f"Received {len(pending_msg)} messages") + + tts_tokenize_msg = [m for m in pending_msg if isinstance(m, TTSTokenizeMsg)] + tts_detokenize_msg = [ + m for m in pending_msg if isinstance(m, TTSDetokenizeMsg) + ] + + # Process TTS tokenize messages + if len(tts_tokenize_msg) > 0: + tensors = [] + for msg in tts_tokenize_msg: + text = msg.text + if text_normalizer is not None and msg.text_normalization: + text = text_normalizer.normalize(text, msg.language) + tensors.append( + tts_prompt_builder.build( + text, + speaking_rate_bucket=msg.speaking_rate_bucket, + quality_buckets=msg.quality_buckets, + ) + ) + user_msgs = [] + for msg, t in zip(tts_tokenize_msg, tensors, strict=True): + speaker_token_position = msg.speaker_token_position + if msg.speaker_embedding is not None: + # Canonical speaker slot is token position 0, matching training. + speaker_slot = tts_prompt_builder.speaker_slot( + dtype=t.dtype, + device=t.device, + ) + t = torch.cat([speaker_slot, t], dim=0) + speaker_token_position = 0 + logger.debug( + "Tokenize uid=%d speaking_rate_bucket=%s quality_buckets=%s " + "text='%s...' frames=%d", + msg.uid, + msg.speaking_rate_bucket, + msg.quality_buckets, + msg.text[:50], + len(t), + ) + user_msgs.append( + TTSUserMsg( + uid=msg.uid, + input_ids=t, + sampling_params=msg.sampling_params, + speaker_embedding=msg.speaker_embedding, + speaker_token_position=speaker_token_position, + clean_speaker_background=msg.clean_speaker_background, + accurate_mode=msg.accurate_mode, + ) + ) + batch_output = BatchTTSBackendMsg( + data=user_msgs + ) + if len(batch_output.data) == 1: + batch_output = batch_output.data[0] + send_backend.put(batch_output) + + # Process TTS detokenize (vocoder) messages + if len(tts_detokenize_msg) > 0: + audio_chunks = tts_vocoder_manager.decode_frames(tts_detokenize_msg) + batch_output = BatchTTSFrontendMsg( + data=[ + TTSAudioReply( + uid=msg.uid, + audio_data=audio, + finished=msg.finished, + ) + for msg, audio in zip( + tts_detokenize_msg, audio_chunks, strict=True + ) + ] + ) + if len(batch_output.data) == 1: + batch_output = batch_output.data[0] + send_frontend.put(batch_output) + + except KeyboardInterrupt: + pass diff --git a/zonos2/tokenizer/textnorm.py b/zonos2/tokenizer/textnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..efaa91c9f273178312b934c14990f1becc31ab34 --- /dev/null +++ b/zonos2/tokenizer/textnorm.py @@ -0,0 +1,172 @@ +"""TTS text normalization using the vendored NeMo forward-TN grammars. + +Converts written text to its spoken form ("$5.32" -> "five dollars thirty two +cents") before tokenization. Grammar construction is expensive (~10-30s per +language) the first time; compiled WFSTs are cached as .far files under +ZONOS2_TTS_NORM_CACHE_DIR (default ~/.cache/zonos2-tts-norm) and reload in +well under a second. Use tools/build_tts_norm_fars.py to prewarm all caches. + +Disable globally with ZONOS2_TTS_NORM=0, or per request via the +text_normalization flag. +""" + +from __future__ import annotations + +import os +import re +import threading +from typing import Dict, Optional + +from zonos2.utils import init_logger + +logger = init_logger(__name__) + +# Server language codes -> NeMo text_normalization language packages. +SERVER_TO_NEMO_LANG: Dict[str, str] = { + "en_us": "en", + "en_gb": "en", + "fr_fr": "fr", + "de": "de", + "es": "es", + "it": "it", + "pt_br": "pt", + "ja": "ja", + "cmn": "zh", + "ko": "ko", +} + +# Upstream's own tests run Korean grammars with lower_cased input; everything +# else uses cased. +LOWER_CASED_LANGS = {"ko"} + +# zh/ja verbalizers read a cached .far but never write one (upstream quirk); +# we post-write it after first compile so later loads are fast. Note ja reads +# a "jp_" prefixed name. +_VERBALIZER_FAR_PREFIX = {"zh": "zh", "ja": "jp"} + +# A digit directly followed by sentence punctuation confuses several upstream +# r1.2.0 grammars: pt's tagger raises FstOpError outright and de reads dates +# digit-by-digit (reproduced with upstream code and its pinned pynini +# 2.1.6.post1 — upstream bugs, not vendoring artifacts). Space the punctuation +# off before normalization; it is re-attached afterwards. +_DIGIT_PUNCT_RE = re.compile(r"(\d)([.!?,;:])(?=\s|$)") +_SPACE_PUNCT_RE = re.compile(r" +([.!?,;:])(?=\s|$)") + +# Moses-based punct_post_process re-attaches punctuation well for these +# languages. For the European languages it also glues currency symbols to the +# following word ("5,32 € am" -> "€am"), so there we skip moses and only +# collapse the spacing we introduced ourselves. +_MOSES_POSTPROCESS_LANGS = {"en", "zh", "ja", "ko"} + + +def default_cache_root() -> str: + return os.environ.get( + "ZONOS2_TTS_NORM_CACHE_DIR", + os.path.expanduser("~/.cache/zonos2-tts-norm"), + ) + + +def normalization_enabled() -> bool: + return os.environ.get("ZONOS2_TTS_NORM", "1") != "0" + + +class TTSTextNormalizer: + """Lazy per-language NeMo normalizers with .far caching. + + Construction and calls are serialized per language: the upstream + Normalizer shares a mutable TokenParser and is not thread-safe. + """ + + def __init__(self, cache_root: str | None = None): + self.cache_root = cache_root or default_cache_root() + self._normalizers: Dict[str, object] = {} + self._locks: Dict[str, threading.Lock] = {} + self._global_lock = threading.Lock() + + @staticmethod + def nemo_lang(language: str) -> Optional[str]: + return SERVER_TO_NEMO_LANG.get(language) + + def supported(self, language: str) -> bool: + return language in SERVER_TO_NEMO_LANG + + def _lang_lock(self, lang: str) -> threading.Lock: + with self._global_lock: + if lang not in self._locks: + self._locks[lang] = threading.Lock() + return self._locks[lang] + + def _build(self, lang: str): + from zonos2.vendor.nemo_text_processing.text_normalization import ( + Normalizer, + ) + + input_case = "lower_cased" if lang in LOWER_CASED_LANGS else "cased" + # One cache dir per (lang, case): upstream .far filenames collide + # across languages (e.g. ja's tagger writes a zh_-prefixed file). + cache_dir = os.path.join(self.cache_root, f"{lang}_{input_case}") + os.makedirs(cache_dir, exist_ok=True) + + logger.info("Loading TTS text normalizer for '%s' (%s)...", lang, input_case) + normalizer = Normalizer( + input_case=input_case, + lang=lang, + cache_dir=cache_dir, + overwrite_cache=False, + ) + + prefix = _VERBALIZER_FAR_PREFIX.get(lang) + if prefix is not None: + far_path = os.path.join( + cache_dir, f"{prefix}_tn_True_deterministic_verbalizer.far" + ) + if not os.path.exists(far_path): + from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + generator_main, + ) + + generator_main(far_path, {"verbalize": normalizer.verbalizer.fst}) + return normalizer + + def get(self, lang: str): + with self._lang_lock(lang): + if lang not in self._normalizers: + self._normalizers[lang] = self._build(lang) + return self._normalizers[lang] + + def warmup(self, languages: list[str] | None = None) -> None: + """Construct normalizers ahead of time (server codes or NeMo codes).""" + langs = languages or sorted(set(SERVER_TO_NEMO_LANG.values())) + for lang in langs: + lang = SERVER_TO_NEMO_LANG.get(lang, lang) + try: + self.get(lang) + except Exception: # noqa: BLE001 + logger.exception("TTS text normalizer warmup failed for '%s'", lang) + + def normalize(self, text: str, language: str) -> str: + """Normalize text for the given server language code. + + Returns the input unchanged for unsupported languages or on any + normalizer error — normalization must never fail a request. + """ + lang = SERVER_TO_NEMO_LANG.get(language) + if lang is None or not text.strip(): + return text + text_in = _DIGIT_PUNCT_RE.sub(r"\1 \2", text) + use_moses = lang in _MOSES_POSTPROCESS_LANGS + try: + normalizer = self.get(lang) + with self._lang_lock(lang): + result = normalizer.normalize(text_in, punct_post_process=use_moses) + except Exception: # noqa: BLE001 + logger.exception( + "TTS text normalization failed for lang=%s; using raw text", language + ) + return text + if isinstance(result, str): + result = _SPACE_PUNCT_RE.sub(r"\1", result) + if not isinstance(result, str) or not result.strip(): + return text + logger.debug("TTS norm [%s]: %r -> %r", language, text, result) + return result diff --git a/zonos2/tokenizer/vocoder.py b/zonos2/tokenizer/vocoder.py new file mode 100644 index 0000000000000000000000000000000000000000..da33afb3de6951b11d1478dceb2963e8f740c9b6 --- /dev/null +++ b/zonos2/tokenizer/vocoder.py @@ -0,0 +1,315 @@ +"""TTS vocoder manager using DAC for audio decoding.""" + +from __future__ import annotations + +from typing import Dict, List + +import numpy as np +import torch +from zonos2.message import TTSDetokenizeMsg + +# Global DAC model cache +_dac_model = None + + +def _get_dac(): + """Lazy load and cache the DAC 44kHz model.""" + global _dac_model + if _dac_model is None: + import dac as dac_module + + _dac_model = ( + dac_module.DAC.load(dac_module.utils.download(model_type="44khz")) + .eval() + .to("cuda") + ) + return _dac_model + + +def shear_up(x: torch.Tensor, pad_id: int) -> torch.Tensor: + """Remove delay: column j shifted up by j rows. + + This is the inverse of shear() - it removes the delay pattern applied + during generation to align all codebook outputs for decoding. + + Args: + x: Input tensor of shape (..., H, W) where H is frames and W is codebooks + pad_id: Padding value to fill empty positions + + Returns: + De-sheared tensor of same shape + """ + H, W = x.shape[-2:] + out = x.new_full(x.shape, pad_id) + for j in range(W): + if H > j: + out[..., : H - j, j] = x[..., j:, j] + return out + + +def decode_dac(codes: torch.Tensor) -> torch.Tensor: + """Decode audio codes using DAC. + + Args: + codes: Tensor of shape (batch, seq_len, n_codebooks) with audio codes + + Returns: + Audio tensor of shape (batch, num_samples) at 44.1kHz + """ + dac = _get_dac() + # Clamp codes to valid range + codes = torch.clamp(codes, max=1023) + # DAC expects (batch, codebooks, seq) + codes = codes.permute(0, 2, 1) + + with torch.no_grad(), torch.inference_mode(): + z = dac.quantizer.from_codes(codes)[0] + audio = dac.decode(z).float().squeeze(1).cpu() + + return audio + + +class TTSVocoderManager: + """Converts audio codebook tokens to PCM audio using DAC. + + Uses streaming-compatible incremental decoding that correctly handles + the shear pattern in multi-codebook audio. The model outputs tokens with + a delay pattern where codebook j is delayed by j frames. shear_up removes + this delay, but requires enough context (n_codebooks frames) to work + correctly. + + For streaming: accumulates frames, waits for enough context, then decodes + incrementally. At end of stream, decodes remaining frames with padding. + """ + + def __init__( + self, + n_codebooks: int = 9, + audio_pad_id: int = 1025, + min_decode_chunk: int = 16, + sample_rate: int = 44100, + overlap_frames: int = 4, + hop_length: int = 512, + ): + """Initialize the vocoder manager. + + Args: + n_codebooks: Number of audio codebooks + audio_pad_id: Audio padding token ID (used in shear_up) + min_decode_chunk: Minimum frames to decode at once (for efficiency) + sample_rate: Output sample rate (44100 for DAC) + overlap_frames: Number of frames to overlap between chunks for OLA + hop_length: DAC audio samples per codebook frame + """ + assert min_decode_chunk > overlap_frames, ( + f"min_decode_chunk ({min_decode_chunk}) must be > " + f"overlap_frames ({overlap_frames})" + ) + self.n_codebooks = n_codebooks + self.audio_pad_id = audio_pad_id + self.min_decode_chunk = min_decode_chunk + self.sample_rate = sample_rate + self.overlap_frames = overlap_frames + self.hop_length = hop_length + + # Per-request state + self._frame_buffers: Dict[int, List[List[int]]] = {} + self._decoded_counts: Dict[int, int] = {} + self._eos_frames: Dict[int, int] = {} + self._overlap_tails: Dict[int, np.ndarray] = {} + + # Crossfade window cache + self._window_cache: Dict[int, np.ndarray] = {} + + def decode_frames(self, msgs: List[TTSDetokenizeMsg]) -> List[bytes]: + """Decode audio frames to PCM chunks with streaming support. + + Accumulates frames and decodes incrementally when enough context is + available. The shear pattern requires n_codebooks-1 extra frames of + context to correctly decode output frames. + + With N accumulated frames, can decode output frames 0 to N-9 completely. + At end of stream, decodes remaining pre-EOS frames. Tail frames + generated for delay alignment are kept as context but are sliced away + before DAC decode. + + Args: + msgs: List of TTS detokenize messages + + Returns: + List of PCM audio bytes (float32, 44.1kHz), one per message + """ + results = [] + for msg in msgs: + uid = msg.uid + + # Initialize state if needed + if uid not in self._frame_buffers: + self._frame_buffers[uid] = [] + self._decoded_counts[uid] = 0 + if msg.eos_frame is not None: + self._eos_frames[uid] = max(0, int(msg.eos_frame)) + + # Accumulate frame + self._frame_buffers[uid].append(msg.audio_codes) + + # Calculate how many complete frames we can decode + total = len(self._frame_buffers[uid]) + # With N frames, can decode output frames 0 to N-(n_codebooks-1)-1 + complete_frames = total - (self.n_codebooks - 1) + if uid in self._eos_frames: + complete_frames = min(complete_frames, self._eos_frames[uid]) + new_decodable = complete_frames - self._decoded_counts[uid] + + should_decode = False + if msg.finished: + # End of stream: decode everything before EOS, then flush any + # withheld overlap tail from the last non-final chunk. + should_decode = ( + complete_frames > self._decoded_counts[uid] + or uid in self._overlap_tails + ) + elif new_decodable >= self.min_decode_chunk: + # Streaming: batch decode when we have enough new frames + should_decode = True + + if should_decode: + audio = self._decode_incremental(uid, msg.finished) + results.append(audio) + else: + results.append(b"") + + # Cleanup finished requests + if msg.finished: + self._frame_buffers.pop(uid, None) + self._decoded_counts.pop(uid, None) + self._eos_frames.pop(uid, None) + self._overlap_tails.pop(uid, None) + + return results + + def _get_crossfade_window(self, length: int) -> np.ndarray: + """Get a cached raised-cosine fade-in window of the given sample length.""" + if length not in self._window_cache: + self._window_cache[length] = ( + 0.5 * (1.0 - np.cos(np.linspace(0, np.pi, length))) + ).astype(np.float32) + return self._window_cache[length] + + def _decode_incremental(self, uid: int, is_final: bool) -> bytes: + """Decode new frames incrementally with OLA crossfading. + + Re-decodes ``overlap_frames`` previously-decoded frames together with + the new frames so DAC has continuous context. The overlapping audio + region is crossfaded with the withheld tail from the previous chunk. + + Each non-final chunk withholds its last ``overlap_frames * hop_length`` + samples (stored in ``_overlap_tails``). These are crossfaded with the + next chunk's head and output at that time. The final chunk outputs all + remaining audio including the tail. + + Args: + uid: Request user ID + is_final: Whether this is the final decode (end of stream) + + Returns: + PCM audio bytes (float32) + """ + buffer = self._frame_buffers[uid] + decoded = self._decoded_counts[uid] + total = len(buffer) + + if is_final: + target_frames = total + else: + target_frames = total - (self.n_codebooks - 1) + if uid in self._eos_frames: + target_frames = min(target_frames, self._eos_frames[uid]) + + if target_frames <= decoded: + if is_final and uid in self._overlap_tails: + return self._overlap_tails.pop(uid).tobytes() + return b"" + + # --- Determine overlap with previous chunk --- + overlap = min(self.overlap_frames, decoded) + + # --- Raw slice including overlap context for shear_up --- + decode_start = decoded - overlap + raw_end = min(target_frames + self.n_codebooks - 1, total) + raw_slice = buffer[decode_start:raw_end] + + # --- shear_up and take output frames --- + device = "cuda" if torch.cuda.is_available() else "cpu" + codes = torch.tensor(raw_slice, dtype=torch.int64, device=device) + codes = shear_up(codes, self.audio_pad_id) + output_count = target_frames - decode_start # overlap + new frames + codes = codes[:output_count].unsqueeze(0) + + # --- DAC decode --- + audio = decode_dac(codes) + audio_np = audio[0].numpy().astype(np.float32) + + # --- Crossfade overlap region with previous tail --- + if overlap > 0 and uid in self._overlap_tails: + prev_tail = self._overlap_tails[uid] + overlap_samples = min( + overlap * self.hop_length, len(prev_tail), len(audio_np) + ) + if overlap_samples > 0: + fade_in = self._get_crossfade_window(overlap_samples) + audio_np[:overlap_samples] = ( + (1.0 - fade_in) * prev_tail[-overlap_samples:] + + fade_in * audio_np[:overlap_samples] + ) + + # --- Withhold tail for next chunk's crossfade (unless final) --- + if not is_final and self.overlap_frames > 0: + tail_samples = min(self.overlap_frames * self.hop_length, len(audio_np)) + self._overlap_tails[uid] = audio_np[-tail_samples:].copy() + output_audio = audio_np[:-tail_samples] if tail_samples < len(audio_np) else np.array([], dtype=np.float32) + else: + output_audio = audio_np + self._overlap_tails.pop(uid, None) + + # --- Advance decoded count by new frames only --- + self._decoded_counts[uid] = target_frames + + return output_audio.tobytes() + + def decode_all( + self, + codes: torch.Tensor, + apply_shear_up: bool = True, + ) -> torch.Tensor: + """Decode a complete sequence of audio codes. + + Args: + codes: Tensor of shape (batch, seq_len, n_codebooks) + apply_shear_up: Whether to apply shear_up before decoding + + Returns: + Audio tensor of shape (batch, num_samples) + """ + if apply_shear_up: + codes = shear_up(codes, self.audio_pad_id) + + codes = codes.to("cuda") + return decode_dac(codes) + + def reset(self, uid: int | None = None): + """Reset frame buffers and decode state. + + Args: + uid: If provided, reset only this request's state. Otherwise reset all. + """ + if uid is not None: + self._frame_buffers.pop(uid, None) + self._decoded_counts.pop(uid, None) + self._eos_frames.pop(uid, None) + self._overlap_tails.pop(uid, None) + else: + self._frame_buffers.clear() + self._decoded_counts.clear() + self._eos_frames.clear() + self._overlap_tails.clear() diff --git a/zonos2/tts/__init__.py b/zonos2/tts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f5a5c655945f4a35d92351943e22ada17b1ed7de --- /dev/null +++ b/zonos2/tts/__init__.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from .llm import TTSLLM +from .prompt import ( + BYTE_TEXT_VOCAB_SIZE, + TTSPromptBuilder, + TTSPromptConfig, + accurate_mode_token_id, + byte_text_vocab_size, + conditioned_text_vocab_size, + quality_token_id, + speaker_background_token_id, + speaking_rate_token_id, + text_to_byte_ids, + text_to_prompt_tokens, + tokens_to_prompt_tokens, +) +from .sequence import TTSSequence + +__all__ = [ + "BYTE_TEXT_VOCAB_SIZE", + "TTSPromptBuilder", + "TTSPromptConfig", + "TTSSequence", + "accurate_mode_token_id", + "byte_text_vocab_size", + "conditioned_text_vocab_size", + "quality_token_id", + "speaker_background_token_id", + "speaking_rate_token_id", + "text_to_byte_ids", + "text_to_prompt_tokens", + "tokens_to_prompt_tokens", + "TTSLLM", +] diff --git a/zonos2/tts/llm.py b/zonos2/tts/llm.py new file mode 100644 index 0000000000000000000000000000000000000000..5942a276be41759826cd9bd798329fc3941f3cee --- /dev/null +++ b/zonos2/tts/llm.py @@ -0,0 +1,398 @@ +"""Offline TTSLLM class for batch TTS generation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List, Tuple + +import torch +from zonos2.distributed import DistributedInfo +from zonos2.message import ( + BaseTTSBackendMsg, + BatchTTSTokenizerMsg, + TTSDetokenizeMsg, + TTSUserMsg, +) +from zonos2.message.tts import TTSSamplingParams +from zonos2.scheduler import SchedulerConfig +from zonos2.scheduler.scheduler import TTSScheduler +from zonos2.tokenizer.vocoder import TTSVocoderManager, shear_up + +from .prompt import TTSPromptBuilder, TTSPromptConfig + + +class RequestAllFinished(Exception): + """Raised when all requests are finished.""" + + pass + + +# Default quality conditioning, matching the server: trailing silence 0.25-0.5s. +DEFAULT_QUALITY_BUCKETS = {"trailing_silence_s": 3} + + +@dataclass +class TTSRequestStatus: + """Status tracking for a TTS request.""" + + uid: int + input_ids: torch.Tensor # 2D (seq_len, frame_width) + output_frames: List[List[int]] # List of audio code frames + eos_frame: int | None = None + finished: bool = False + + +class TTSLLM(TTSScheduler): + """TTS-specific LLM interface for offline audio generation. + + This class provides a simplified interface for TTS generation with: + - Text or pre-tokenized prompt input + - Multi-codebook sampling with repetition penalty and top-k/top-p/min-p + - EOS detection with frame alignment + - Batch generation support + - Optional audio decoding with DAC + + Example usage: + tts = TTSLLM(model_path="/path/to/model") + + # Generate from text + results = tts.generate(["Hello world"], TTSSamplingParams()) + + # Results contain audio tokens and optionally decoded audio + for r in results: + print(r["audio_tokens"]) # List of frames + if r["audio"]: + # r["audio"] is PCM bytes at 44.1kHz + pass + """ + + def __init__( + self, + model_path: str, + dtype: torch.dtype = torch.bfloat16, + n_codebooks: int | None = None, + codebook_size: int | None = None, + text_vocab: int | None = None, + eoa_id: int | None = None, + audio_pad_id: int | None = None, + decode_audio: bool = True, + **kwargs, + ): + """Initialize TTSLLM. + + Args: + model_path: Path to the model checkpoint + dtype: Data type for model (default: bfloat16) + n_codebooks: Number of audio codebooks (optional, auto-detected from model) + codebook_size: Size of each codebook vocabulary (optional, auto-detected from model) + text_vocab: Text vocabulary size (optional, auto-detected from model) + eoa_id: End-of-audio token ID (optional, auto-detected from model) + audio_pad_id: Audio padding token ID (optional, auto-detected from model) + decode_audio: Whether to decode audio using DAC (default: True) + **kwargs: Additional arguments passed to scheduler + """ + config = SchedulerConfig( + model_path=model_path, + tp_info=DistributedInfo(0, 1), + dtype=dtype, + offline_mode=True, + **kwargs, + ) + super().__init__(config) + + # Override TTS settings only if explicitly provided (otherwise use model config values) + if n_codebooks is not None: + self.n_codebooks = n_codebooks + if codebook_size is not None: + self.codebook_size = codebook_size + if eoa_id is not None: + self.eoa_id = eoa_id + if audio_pad_id is not None: + self.audio_pad_id = audio_pad_id + if text_vocab is not None: + self.text_vocab = text_vocab + + self.decode_audio = decode_audio + + # Request tracking + self.pending_requests: List[Tuple[torch.Tensor, TTSSamplingParams]] = [] + self.status_map: Dict[int, TTSRequestStatus] = {} + self.counter = 0 + + # Optional vocoder for audio decoding + if decode_audio: + self._vocoder = TTSVocoderManager( + n_codebooks=n_codebooks, + audio_pad_id=audio_pad_id, + ) + else: + self._vocoder = None + self._prompt_builder = TTSPromptBuilder( + TTSPromptConfig( + n_codebooks=self.n_codebooks, + audio_pad_id=self.audio_pad_id, + text_vocab=self.text_vocab, + speaking_rate_num_buckets=self.speaking_rate_num_buckets, + quality_bucket_counts=tuple(self.quality_bucket_counts), + speaker_background_num_buckets=self.speaker_background_num_buckets, + accurate_mode_num_buckets=self.accurate_mode_num_buckets, + # Match the server prompt format: trained prompts end with a + # short silence prefix before audio generation begins. + prepend_silence=True, + ) + ) + + def _resolve_quality_buckets( + self, quality_buckets: Dict[str, int | None] | List[int | None] | None + ) -> List[int | None] | None: + """Map a quality bucket dict/list onto the model's feature order. + + None applies the default conditioning (same as the server); pass an + empty dict or list to disable quality tokens entirely. + """ + if not self.quality_bucket_counts or sum(self.quality_bucket_counts) == 0: + return None + if quality_buckets is None: + quality_buckets = DEFAULT_QUALITY_BUCKETS + if isinstance(quality_buckets, dict): + return [quality_buckets.get(feature) for feature in self.quality_features] + resolved = list(quality_buckets)[: len(self.quality_features)] + resolved += [None] * (len(self.quality_features) - len(resolved)) + return resolved + + def _tokenize_one( + self, + prompt: str | List[List[int]], + speaking_rate_bucket: int | None = None, + quality_buckets: Dict[str, int | None] | List[int | None] | None = None, + ) -> torch.Tensor: + """Convert a prompt to 2D token tensor. + + Args: + prompt: Text string or pre-tokenized unpacked tokens + + Returns: + 2D tensor of shape (seq_len, frame_width) + """ + if isinstance(prompt, str): + return self._prompt_builder.build( + prompt, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=self._resolve_quality_buckets(quality_buckets), + ) + else: + prompt_ids = prompt + + return torch.tensor(prompt_ids, dtype=torch.int32, device="cpu") + + def offline_receive_msg(self, blocking: bool = False) -> List[BaseTTSBackendMsg]: + """Receive messages from the pending queue.""" + if blocking and len(self.pending_requests) == 0: + raise RequestAllFinished() + + results: List[BaseTTSBackendMsg] = [] + added, sum_input_len = 0, 0 + + for i, (input_ids, sampling_params) in enumerate(self.pending_requests): + if sum_input_len >= self.prefill_budget: + break + + input_len = len(input_ids) + sum_input_len += input_len + uid = self.counter + added + added += 1 + + results.append( + TTSUserMsg( + uid=uid, + input_ids=input_ids, + sampling_params=sampling_params, + ) + ) + + self.status_map[uid] = TTSRequestStatus( + uid=i, # Map back to original index + input_ids=input_ids, + output_frames=[], + ) + + self.counter += added + self.pending_requests = self.pending_requests[added:] + return results + + def offline_send_result(self, reply: BatchTTSTokenizerMsg) -> None: + """Process results from the scheduler.""" + for msg in reply.data: + assert isinstance(msg, TTSDetokenizeMsg) + status = self.status_map[msg.uid] + + if not msg.finished: + status.output_frames.append(msg.audio_codes) + else: + status.finished = True + status.eos_frame = msg.eos_frame + + def generate( + self, + prompts: List[str] | List[List[List[int]]], + sampling_params: TTSSamplingParams | List[TTSSamplingParams], + decode_audio: bool | None = None, + speaking_rate_bucket: int | List[int | None] | None = None, + quality_buckets: Dict[str, int | None] | List[int | None] | None = None, + ) -> List[Dict]: + """Generate audio tokens for a batch of prompts. + + Args: + prompts: List of text strings or pre-tokenized prompts + sampling_params: Sampling parameters (single or per-prompt) + decode_audio: Override instance setting for audio decoding + speaking_rate_bucket: Optional bucket index, or one bucket per prompt. + quality_buckets: Quality bucket indices, keyed by feature name or as + a list in the model's feature order. None applies the default + conditioning; pass {} to disable quality tokens. + + Returns: + List of dicts with: + - "audio_tokens": List of generated frames [[cb0, ..., cb8], ...] + - "eos_frame": Frame index where EOS was detected (or None) + - "audio": PCM audio bytes if decode_audio=True, else None + - "sample_rate": Audio sample rate (44100) + """ + # Reset state + self.pending_requests = [] + self.status_map = {} + self.counter = 0 + + # Normalize sampling_params + if isinstance(sampling_params, TTSSamplingParams): + sampling_params = [sampling_params] * len(prompts) + if isinstance(speaking_rate_bucket, list): + speaking_rate_buckets = speaking_rate_bucket + else: + speaking_rate_buckets = [speaking_rate_bucket] * len(prompts) + + # Tokenize and queue all requests + for prompt, sp, rate_bucket in zip( + prompts, + sampling_params, + speaking_rate_buckets, + ): + input_ids = self._tokenize_one( + prompt, + speaking_rate_bucket=rate_bucket, + quality_buckets=quality_buckets, + ) + self.pending_requests.append((input_ids, sp)) + + # Run generation + try: + self.run_forever() + except RequestAllFinished: + pass + + # Determine audio decoding + should_decode = decode_audio if decode_audio is not None else self.decode_audio + + # Collect results in order + results = [] + for i in range(len(prompts)): + status = self.status_map[i] + audio_tokens = status.output_frames + + # Decode audio if requested + audio_bytes = None + if should_decode and audio_tokens and self._vocoder: + # Convert to tensor, align delayed codebooks, then drop EOS and + # post-EOS frames before DAC decode. + codes = torch.tensor(audio_tokens, dtype=torch.int64, device="cuda") + codes = shear_up(codes, self.audio_pad_id) + if status.eos_frame is not None: + codes = codes[: max(0, status.eos_frame)] + if codes.numel() == 0: + results.append( + { + "audio_tokens": audio_tokens, + "eos_frame": status.eos_frame, + "audio": b"", + "sample_rate": 44100, + } + ) + continue + codes = codes.unsqueeze(0) # Add batch dim + + # Decode with vocoder + audio = self._vocoder.decode_all(codes, apply_shear_up=False) + audio_bytes = audio[0].numpy().astype("float32").tobytes() + + results.append( + { + "audio_tokens": audio_tokens, + "eos_frame": status.eos_frame, + "audio": audio_bytes, + "sample_rate": 44100, + } + ) + + return results + + def generate_one( + self, + prompt: str | List[List[int]], + sampling_params: TTSSamplingParams | None = None, + decode_audio: bool | None = None, + speaking_rate_bucket: int | None = None, + quality_buckets: Dict[str, int | None] | List[int | None] | None = None, + ) -> Dict: + """Generate audio for a single prompt. + + Convenience method that wraps generate() for single inputs. + + Args: + prompt: Text string or pre-tokenized prompt + sampling_params: Sampling parameters (default: TTSSamplingParams()) + decode_audio: Override instance setting for audio decoding + speaking_rate_bucket: Optional speaking-rate conditioning bucket. + quality_buckets: Quality bucket indices by feature name; None applies + the default conditioning, {} disables quality tokens. + + Returns: + Dict with audio_tokens, eos_frame, audio, sample_rate + """ + if sampling_params is None: + sampling_params = TTSSamplingParams() + + results = self.generate( + [prompt], + sampling_params, + decode_audio=decode_audio, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=quality_buckets, + ) + return results[0] + + def save_audio( + self, + audio_bytes: bytes, + path: str, + sample_rate: int = 44100, + ) -> None: + """Save PCM audio bytes to a WAV file. + + Args: + audio_bytes: PCM audio data (float32) + path: Output file path + sample_rate: Audio sample rate + """ + import wave + + import numpy as np + + audio = np.frombuffer(audio_bytes, dtype=np.float32) + # Convert to int16 for WAV + audio_int16 = (audio * 32767).astype(np.int16) + + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) # 16-bit + f.setframerate(sample_rate) + f.writeframes(audio_int16.tobytes()) diff --git a/zonos2/tts/prompt.py b/zonos2/tts/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac7d780316722fa3df4c031713734acddba3c85 --- /dev/null +++ b/zonos2/tts/prompt.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +PAD_ID, UNK_ID, BOS_ID, EOS_ID = 0, 1, 2, 3 +SPECIAL_TOKEN_IDS = (PAD_ID, UNK_ID, BOS_ID, EOS_ID) + +# 192 legacy symbol IDs followed by 256 byte IDs. +# With 21 speaking-rate buckets, text_vocab is 469: +# bytes occupy 192..447, rate buckets occupy 448..468, and 469 is text padding. +LEGACY_SYMBOL_VOCAB_SIZE = 192 +BYTE_VOCAB_SIZE = 256 +BYTE_TEXT_VOCAB_SIZE = LEGACY_SYMBOL_VOCAB_SIZE + BYTE_VOCAB_SIZE + + +# Pre-computed silence tokens for 0.2s at 44.1kHz (17 frames x 9 codebooks). +_SILENCE_TOKENS_0_2S = [ + [568, 778, 338, 524, 967, 360, 728, 550, 90], + [568, 778, 10, 674, 364, 981, 741, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 804, 10, 674, 364, 981, 568, 378, 731], + [568, 778, 721, 842, 264, 974, 989, 507, 308], +] + + +@dataclass(frozen=True) +class TTSPromptConfig: + n_codebooks: int = 9 + audio_pad_id: int = 1025 + text_vocab: int = BYTE_TEXT_VOCAB_SIZE + speaking_rate_num_buckets: int = 0 + quality_bucket_counts: tuple[int, ...] = () + speaker_background_num_buckets: int = 0 + accurate_mode_num_buckets: int = 0 + prepend_silence: bool = True + + def __post_init__(self) -> None: + n_codebooks = int(self.n_codebooks) + audio_pad_id = int(self.audio_pad_id) + if self.text_vocab is None: + raise ValueError("text_vocab is required for TTS prompt construction.") + text_vocab = int(self.text_vocab) + speaking_rate_num_buckets = int(self.speaking_rate_num_buckets) + quality_bucket_counts = _normalize_quality_bucket_counts(self.quality_bucket_counts) + speaker_background_num_buckets = int(self.speaker_background_num_buckets) + accurate_mode_num_buckets = int(self.accurate_mode_num_buckets) + + if n_codebooks <= 0: + raise ValueError("n_codebooks must be positive.") + if audio_pad_id < 0: + raise ValueError("audio_pad_id must be non-negative.") + if speaking_rate_num_buckets < 0: + raise ValueError("speaking_rate_num_buckets must be non-negative.") + if speaker_background_num_buckets < 0: + raise ValueError("speaker_background_num_buckets must be non-negative.") + if accurate_mode_num_buckets < 0: + raise ValueError("accurate_mode_num_buckets must be non-negative.") + + if text_vocab < BYTE_TEXT_VOCAB_SIZE: + raise ValueError( + "text_vocab must include the byte vocabulary; " + f"expected at least {BYTE_TEXT_VOCAB_SIZE}, got {text_vocab}." + ) + # Validates that all conditioning buckets fit inside text_vocab. + _conditioning_base_text_vocab( + text_vocab, + speaking_rate_num_buckets, + quality_bucket_counts, + speaker_background_num_buckets, + accurate_mode_num_buckets, + context="TTS prompt construction", + ) + + object.__setattr__(self, "n_codebooks", n_codebooks) + object.__setattr__(self, "audio_pad_id", audio_pad_id) + object.__setattr__(self, "text_vocab", text_vocab) + object.__setattr__(self, "speaking_rate_num_buckets", speaking_rate_num_buckets) + object.__setattr__(self, "quality_bucket_counts", quality_bucket_counts) + object.__setattr__(self, "speaker_background_num_buckets", speaker_background_num_buckets) + object.__setattr__(self, "accurate_mode_num_buckets", accurate_mode_num_buckets) + + +def byte_text_vocab_size() -> int: + return BYTE_TEXT_VOCAB_SIZE + + +def conditioned_text_vocab_size( + speaking_rate_num_buckets: int = 0, + quality_num_buckets: int = 0, + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> int: + counts = ( + int(speaking_rate_num_buckets), + int(quality_num_buckets), + int(speaker_background_num_buckets), + int(accurate_mode_num_buckets), + ) + if any(count < 0 for count in counts): + raise ValueError("conditioning bucket counts must be non-negative.") + return BYTE_TEXT_VOCAB_SIZE + sum(counts) + + +def text_to_byte_ids(text: str) -> list[int]: + return [BOS_ID, *(byte + LEGACY_SYMBOL_VOCAB_SIZE for byte in text.encode("utf-8")), EOS_ID] + + +def _normalize_quality_bucket_counts(quality_bucket_counts) -> tuple[int, ...]: + counts = tuple(int(count) for count in (quality_bucket_counts or ())) + if any(count < 0 for count in counts): + raise ValueError("quality_bucket_counts must be non-negative.") + return counts + + +def _conditioning_base_text_vocab( + text_vocab: int | None, + speaking_rate_num_buckets: int, + quality_bucket_counts=(), + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, + *, + context: str, +) -> int: + """First conditioning token id; everything below it is normal text vocabulary. + + Conditioning tokens occupy the tail of the text vocabulary in this order: + speaking-rate buckets, quality buckets (per feature), speaker-background + markers (clean, noisy), accurate-mode marker. text_vocab itself is padding. + """ + if text_vocab is None: + raise ValueError(f"text_vocab is required for {context}.") + + counts = _normalize_quality_bucket_counts(quality_bucket_counts) + base_text_vocab = ( + int(text_vocab) + - int(speaking_rate_num_buckets) + - sum(counts) + - int(speaker_background_num_buckets) + - int(accurate_mode_num_buckets) + ) + if base_text_vocab < 0: + raise ValueError( + "text_vocab is smaller than the configured conditioning buckets; " + "cannot locate conditioning tokens." + ) + return base_text_vocab + + +def speaking_rate_token_id( + text_vocab: int | None, + speaking_rate_num_buckets: int, + speaking_rate_bucket: int, + quality_bucket_counts=(), + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> int: + num_buckets = int(speaking_rate_num_buckets) + if num_buckets <= 0: + raise ValueError("Current model does not define speaking-rate buckets.") + + bucket = int(speaking_rate_bucket) + if bucket < 0 or bucket >= num_buckets: + raise ValueError(f"speaking_rate_bucket must be in [0, {num_buckets - 1}], got {bucket}.") + + base_text_vocab = _conditioning_base_text_vocab( + text_vocab, + num_buckets, + quality_bucket_counts, + speaker_background_num_buckets, + accurate_mode_num_buckets, + context="speaking-rate conditioning", + ) + return base_text_vocab + bucket + + +def quality_token_id( + text_vocab: int | None, + speaking_rate_num_buckets: int, + quality_bucket_counts, + feature_idx: int, + quality_bucket: int, + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> int: + counts = _normalize_quality_bucket_counts(quality_bucket_counts) + if not counts: + raise ValueError("Current model does not define quality buckets.") + + feature = int(feature_idx) + if feature < 0 or feature >= len(counts): + raise ValueError(f"quality feature index must be in [0, {len(counts) - 1}], got {feature}.") + + num_buckets = counts[feature] + if num_buckets <= 0: + raise ValueError(f"quality feature {feature} does not define buckets.") + + bucket = int(quality_bucket) + if bucket < 0 or bucket >= num_buckets: + raise ValueError( + f"quality bucket for feature {feature} must be in [0, {num_buckets - 1}], got {bucket}." + ) + + base_text_vocab = _conditioning_base_text_vocab( + text_vocab, + speaking_rate_num_buckets, + counts, + speaker_background_num_buckets, + accurate_mode_num_buckets, + context="quality conditioning", + ) + return ( + base_text_vocab + + int(speaking_rate_num_buckets) + + sum(counts[:feature]) + + bucket + ) + + +def speaker_background_token_id( + text_vocab: int | None, + speaking_rate_num_buckets: int, + quality_bucket_counts, + clean: bool, + speaker_background_num_buckets: int = 2, + accurate_mode_num_buckets: int = 0, +) -> int: + """Token id for the clean/noisy speaker-background marker.""" + num_buckets = int(speaker_background_num_buckets) + if num_buckets < 2: + raise ValueError( + "speaker_background_num_buckets must be at least 2 for background tokens." + ) + + counts = _normalize_quality_bucket_counts(quality_bucket_counts) + base_text_vocab = _conditioning_base_text_vocab( + text_vocab, + speaking_rate_num_buckets, + counts, + num_buckets, + accurate_mode_num_buckets, + context="speaker-background conditioning", + ) + return ( + base_text_vocab + + int(speaking_rate_num_buckets) + + sum(counts) + + (0 if bool(clean) else 1) + ) + + +def accurate_mode_token_id( + text_vocab: int | None, + speaking_rate_num_buckets: int, + quality_bucket_counts, + speaker_background_num_buckets: int = 2, + accurate_mode_num_buckets: int = 1, +) -> int: + """Token id for the accurate-mode marker (absent = expressive mode).""" + accurate_count = int(accurate_mode_num_buckets) + if accurate_count <= 0: + raise ValueError("accurate_mode_num_buckets must be positive.") + background_count = int(speaker_background_num_buckets) + if background_count < 2: + raise ValueError( + "speaker_background_num_buckets must be at least 2 for accurate-mode tokens." + ) + + counts = _normalize_quality_bucket_counts(quality_bucket_counts) + base_text_vocab = _conditioning_base_text_vocab( + text_vocab, + speaking_rate_num_buckets, + counts, + background_count, + accurate_count, + context="accurate-mode conditioning", + ) + return ( + base_text_vocab + + int(speaking_rate_num_buckets) + + sum(counts) + + background_count + ) + + +def tokens_to_prompt_tokens( + tokens: list[int], + n_codebooks: int = 9, + audio_pad_id: int = 1025, + text_vocab: int | None = None, + speaking_rate_num_buckets: int = 0, + speaking_rate_bucket: int | None = None, + quality_bucket_counts=(), + quality_buckets=None, + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> list[list[int]]: + quality_bucket_counts = _normalize_quality_bucket_counts(quality_bucket_counts) + if text_vocab is None: + text_vocab = conditioned_text_vocab_size( + speaking_rate_num_buckets, + sum(quality_bucket_counts), + speaker_background_num_buckets, + accurate_mode_num_buckets, + ) + config = TTSPromptConfig( + n_codebooks=n_codebooks, + audio_pad_id=audio_pad_id, + text_vocab=text_vocab, + speaking_rate_num_buckets=speaking_rate_num_buckets, + quality_bucket_counts=quality_bucket_counts, + speaker_background_num_buckets=speaker_background_num_buckets, + accurate_mode_num_buckets=accurate_mode_num_buckets, + prepend_silence=False, + ) + return _text_rows( + tokens, + config, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=quality_buckets, + ) + + +def text_to_prompt_tokens( + text: str, + n_codebooks: int = 9, + audio_pad_id: int = 1025, + text_vocab: int | None = None, + speaking_rate_num_buckets: int = 0, + speaking_rate_bucket: int | None = None, + quality_bucket_counts=(), + quality_buckets=None, + speaker_background_num_buckets: int = 0, + accurate_mode_num_buckets: int = 0, +) -> list[list[int]]: + return tokens_to_prompt_tokens( + text_to_byte_ids(text), + n_codebooks=n_codebooks, + audio_pad_id=audio_pad_id, + text_vocab=text_vocab, + speaking_rate_num_buckets=speaking_rate_num_buckets, + speaking_rate_bucket=speaking_rate_bucket, + quality_bucket_counts=quality_bucket_counts, + quality_buckets=quality_buckets, + speaker_background_num_buckets=speaker_background_num_buckets, + accurate_mode_num_buckets=accurate_mode_num_buckets, + ) + + +def shear(x: torch.Tensor, pad: int) -> torch.Tensor: + T, C = x.shape + padded = x.new_full((C - 1 + T, C), pad) + padded[C - 1 :] = x + row_idx = (C - 1) + torch.arange(T, device=x.device).unsqueeze(1) - torch.arange( + C, device=x.device + ) + return padded.gather(0, row_idx) + + +def silence_prompt_tokens(config: TTSPromptConfig) -> torch.Tensor: + silence = torch.tensor(_SILENCE_TOKENS_0_2S, dtype=torch.int32) + sheared = shear(silence[:, : config.n_codebooks], config.audio_pad_id) + text_col = torch.full((sheared.shape[0], 1), config.text_vocab, dtype=torch.int32) + return torch.cat([sheared, text_col], dim=1) + + +def make_speaker_slot( + config: TTSPromptConfig, + *, + dtype: torch.dtype = torch.int32, + device: torch.device | str | None = None, +) -> torch.Tensor: + slot = torch.full((1, config.n_codebooks + 1), config.audio_pad_id, dtype=dtype, device=device) + slot[:, config.n_codebooks] = config.text_vocab + return slot + + +def _text_rows( + tokens: list[int], + config: TTSPromptConfig, + *, + speaking_rate_bucket: int | None = None, + quality_buckets=None, +) -> list[list[int]]: + rows: list[list[int]] = [] + if speaking_rate_bucket is not None: + rows.append( + [config.audio_pad_id] * config.n_codebooks + + [ + speaking_rate_token_id( + config.text_vocab, + config.speaking_rate_num_buckets, + speaking_rate_bucket, + config.quality_bucket_counts, + config.speaker_background_num_buckets, + config.accurate_mode_num_buckets, + ) + ] + ) + + if quality_buckets is not None: + for feature_idx, bucket in enumerate(quality_buckets): + if bucket is None: + continue + rows.append( + [config.audio_pad_id] * config.n_codebooks + + [ + quality_token_id( + config.text_vocab, + config.speaking_rate_num_buckets, + config.quality_bucket_counts, + feature_idx, + bucket, + config.speaker_background_num_buckets, + config.accurate_mode_num_buckets, + ) + ] + ) + + rows.extend([config.audio_pad_id] * config.n_codebooks + [token] for token in tokens) + return rows + + +class TTSPromptBuilder: + def __init__(self, config: TTSPromptConfig): + self.config = config + self._silence_tokens = silence_prompt_tokens(config) if config.prepend_silence else None + + def build_text_prompt( + self, + text: str, + *, + speaking_rate_bucket: int | None = None, + quality_buckets=None, + ) -> torch.Tensor: + rows = _text_rows( + text_to_byte_ids(text), + self.config, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=quality_buckets, + ) + return torch.tensor(rows, dtype=torch.int32) + + def build( + self, + text: str, + *, + speaking_rate_bucket: int | None = None, + quality_buckets=None, + ) -> torch.Tensor: + prompt = self.build_text_prompt( + text, + speaking_rate_bucket=speaking_rate_bucket, + quality_buckets=quality_buckets, + ) + if self._silence_tokens is not None: + prompt = torch.cat([prompt, self._silence_tokens], dim=0) + return prompt + + def speaker_slot( + self, + *, + dtype: torch.dtype = torch.int32, + device: torch.device | str | None = None, + ) -> torch.Tensor: + return make_speaker_slot(self.config, dtype=dtype, device=device) diff --git a/zonos2/tts/sampler.py b/zonos2/tts/sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..575e69b3297e237016681495e42148101f949c58 --- /dev/null +++ b/zonos2/tts/sampler.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import logging +from typing import List + +import torch +import torch.nn.functional as F +from zonos2.utils import init_logger + +logger = init_logger(__name__) + +EOA_TOKEN = 1024 # End of audio token + + +def apply_top_p(probs: torch.Tensor, p: float) -> torch.Tensor: + """Apply nucleus (top-p) sampling to probabilities.""" + if p <= 0.0 or p >= 1.0: + return probs + probs_sort, probs_idx = torch.sort(probs, dim=-1, descending=True) + probs_sum = torch.cumsum(probs_sort, dim=-1) + mask = probs_sum - probs_sort > p + probs_sort = probs_sort.masked_fill(mask, 0.0) + probs = probs.scatter(-1, probs_idx, probs_sort) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp(min=1e-8) + return probs + + +def apply_min_p(probs: torch.Tensor, min_p: float) -> torch.Tensor: + """Apply min-p filtering to probabilities.""" + if min_p <= 0.0: + return probs + top_probs, _ = probs.max(dim=-1, keepdim=True) + tokens_to_remove = probs < (min_p * top_probs) + probs = probs.masked_fill(tokens_to_remove, 0.0) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp(min=1e-8) + return probs + + +def apply_repetition_penalty( + logits: torch.Tensor, + repetition_token_ids: torch.Tensor | None, + repetition_penalties: torch.Tensor | None, +) -> torch.Tensor: + """Apply a per-sequence repetition penalty to per-codebook logits. + + repetition_token_ids is shaped (B, n_codebooks, window). A token id in one + codebook only penalizes the matching codebook's logits. + """ + if repetition_token_ids is None or repetition_penalties is None: + return logits + if repetition_token_ids.numel() == 0: + return logits + + B, C, V = logits.shape + safe_token_ids = repetition_token_ids.clamp(min=0, max=V - 1).long() + valid = (repetition_token_ids >= 0) & (repetition_token_ids < V) + + counts = torch.zeros((B, C, V), dtype=torch.int32, device=logits.device) + counts.scatter_add_(-1, safe_token_ids, valid.to(torch.int32)) + repeated = counts > 0 + + penalties = repetition_penalties.view(B, 1, 1).clamp(min=1.0) + adjusted_logits = torch.where(logits > 0, logits / penalties, logits * penalties) + return torch.where(repeated, adjusted_logits, logits) + + +_debug_sample_count = 0 + + +def _compute_logit_stats_debug(logits: torch.Tensor) -> dict: + """Compute per-codebook stats matching reference format. + + Args: + logits: (B, n_codebooks, vocab) - uses first batch element + + Returns: + Dict with per-codebook stats: entropy, top1_prob, margin, top5_ids + """ + lg = logits[0].float() # (n_codebooks, vocab) + + probs = F.softmax(lg, dim=-1) + log_probs = F.log_softmax(lg, dim=-1) + + # Entropy: -sum(p * log(p)) + entropy = -(probs * log_probs).sum(dim=-1) + + # Top-k probs and ids + sorted_probs, sorted_indices = probs.sort(dim=-1, descending=True) + top1_prob = sorted_probs[:, 0] + top5_ids = sorted_indices[:, :5] + + # Logit margin (top1 - top2) + sorted_logits, _ = lg.sort(dim=-1, descending=True) + margin = sorted_logits[:, 0] - sorted_logits[:, 1] + + return { + "entropy": entropy.tolist(), + "top1_prob": top1_prob.tolist(), + "margin": margin.tolist(), + "top5_ids": top5_ids.tolist(), + } + + +def sample_tts( + logits: torch.Tensor, + temperatures: torch.Tensor, + top_ks: torch.Tensor, + top_ps: torch.Tensor, + min_ps: torch.Tensor, + repetition_token_ids: torch.Tensor | None = None, + repetition_penalties: torch.Tensor | None = None, + text_vocab: int = 0, + generators: list[torch.Generator | None] | None = None, +) -> List[List[int]]: + """Functional interface for TTS sampling. + + Args: + logits: Shape (B, n_codebooks, vocab_size) + temperatures: Per-sequence temperatures (B,) + top_ks: Per-sequence top-k values (B,) + top_ps: Per-sequence top-p values (B,) + min_ps: Per-sequence min-p values (B,) + repetition_token_ids: Recent token ids per sequence/codebook (B, C, W) + repetition_penalties: Per-sequence repetition penalties (B,), 1.0 disables + text_vocab: Text vocabulary size (appended as placeholder) + + Returns: + List of unpacked tokens, each is [cb0, cb1, ..., cb8, text_placeholder] + """ + global _debug_sample_count + B, C, V = logits.shape + + # Debug: print detailed stats matching reference format for first 10 samples + # Guard with is_current_stream_capturing to avoid CUDA graph capture errors + if (logger.isEnabledFor(logging.DEBUG) + and _debug_sample_count < 10 + and EOA_TOKEN < V + and not torch.cuda.is_current_stream_capturing()): + _debug_sample_count += 1 + step = _debug_sample_count + + stats = _compute_logit_stats_debug(logits) + + logger.debug("Step %d per-codebook stats:", step) + for cb in range(C): + top5 = stats["top5_ids"][cb] + logger.debug( + " CB%d: entropy=%.4f, top1_prob=%.4f, top1_id=%d, margin=%.2f, top5=%s", + cb, stats["entropy"][cb], stats["top1_prob"][cb], + top5[0], stats["margin"][cb], top5, + ) + + probs_raw = F.softmax(logits[0], dim=-1) + eoa_probs = probs_raw[:, EOA_TOKEN].tolist() + eoa_logits = logits[0, :, EOA_TOKEN].tolist() + logger.debug( + "Step %d EOA(1024) logits: [%s]", + step, ", ".join(f"{x:.2f}" for x in eoa_logits), + ) + logger.debug( + "Step %d EOA(1024) probs: [%s]", + step, ", ".join(f"{x:.4f}" for x in eoa_probs), + ) + + logits = apply_repetition_penalty( + logits, + repetition_token_ids=repetition_token_ids, + repetition_penalties=repetition_penalties, + ) + + # Check if all greedy (temperature <= 0) + if (temperatures <= 0).all(): + next_ids = torch.argmax(logits, dim=-1) + else: + # Per-sequence temperature scaling: (B,) -> (B, 1, 1) for broadcasting + temp_broadcast = temperatures.view(B, 1, 1).clamp(min=1e-8) + logits = logits / temp_broadcast + logits_flat = logits.view(B * C, V) + + # Use min top_k across batch for efficient single topk call + top_k = int(top_ks.min().item()) + if 0 < top_k < V: + values, _ = torch.topk(logits_flat, top_k, dim=-1) + kth = values[..., -1].unsqueeze(-1) + mask = logits_flat < kth + logits_flat = logits_flat.masked_fill(mask, float("-inf")) + + probs = F.softmax(logits_flat, dim=-1) + + # Apply top_p (use max across batch - more restrictive) + top_p = float(top_ps.max().item()) + if 0.0 < top_p < 1.0: + probs = apply_top_p(probs, top_p) + + # Apply min_p (use max across batch - more restrictive) + min_p = float(min_ps.max().item()) + if min_p > 0.0: + probs = apply_min_p(probs, min_p) + + # An aggressive min_p (>1) or top_p can remove every token in a row, + # leaving an all-zero distribution. torch.multinomial would then raise a + # CUDA device-side assert that aborts the scheduler, so fall back to + # greedy (argmax) for any fully-masked row. + invalid_rows = probs.sum(dim=-1) <= 0 + if bool(invalid_rows.any()): + greedy = logits_flat.argmax(dim=-1) + fallback = torch.zeros_like(probs) + fallback.scatter_(-1, greedy.unsqueeze(-1), 1.0) + probs = torch.where(invalid_rows.unsqueeze(-1), fallback, probs) + + # Use per-request generators for deterministic sampling when seeds are set + has_generators = generators and any(g is not None for g in generators) + if has_generators and B == 1 and generators[0] is not None: + # Fast path: single request with a generator + next_ids = torch.multinomial( + probs, num_samples=1, generator=generators[0] + ).view(B, C) + elif has_generators: + # Batched with mixed generators: sample per-row + # probs is (B*C, V), each request owns C consecutive rows + rows = [] + for i in range(B): + row_probs = probs[i * C : (i + 1) * C] # (C, V) + gen = generators[i] if i < len(generators) else None + rows.append( + torch.multinomial(row_probs, num_samples=1, generator=gen) + ) + next_ids = torch.cat(rows, dim=0).view(B, C) + else: + next_ids = torch.multinomial(probs, num_samples=1).view(B, C) + + # Return unpacked format: list of (audio codes + text placeholder) + next_ids_list = next_ids.tolist() + + # Debug: print sampled tokens for first 10 samples + if (logger.isEnabledFor(logging.DEBUG) + and _debug_sample_count <= 10 + and not torch.cuda.is_current_stream_capturing()): + step = _debug_sample_count + sampled = next_ids_list[0] if next_ids_list else [] + logger.debug("Step %d sampled: %s", step, sampled) + for i, row in enumerate(next_ids_list): + eoa_positions = [j for j, t in enumerate(row) if t == EOA_TOKEN] + if eoa_positions: + logger.debug( + "Step %d *** EOA SAMPLED at CB %s for batch %d ***", + step, eoa_positions, i, + ) + + return [row + [text_vocab] for row in next_ids_list] diff --git a/zonos2/tts/sequence.py b/zonos2/tts/sequence.py new file mode 100644 index 0000000000000000000000000000000000000000..7576cfec5b25650768f3d665f1cfe42639aedb25 --- /dev/null +++ b/zonos2/tts/sequence.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional + +from zonos2.message.tts import TTSSamplingParams + + +@dataclass +class TTSSequence: + """Sequence tracking for TTS generation with EOS detection. + + Token format: Each token is a list of [cb0, cb1, ..., cb8, text_token] + (unpacked format with n_codebooks audio codes + 1 text token) + + EOS Detection: + - When EOS token (eoa_id) is detected in any delayed codebook, + we record the aligned eos_frame and start a countdown. + - The countdown allows n_codebooks + 1 additional steps for frame alignment. + - Sequence is finished when countdown reaches 0 after EOS detection. + """ + + prompt_ids: List[List[int]] + sampling_params: TTSSamplingParams + seq_id: int = 0 + token_ids: List[List[int]] = field(default_factory=list) + eos_frame: Optional[int] = None + eos_countdown: int = 0 + n_codebooks: int = 9 + eoa_id: int = 1024 + + def __post_init__(self): + # Initialize token_ids with prompt + self.token_ids = list(self.prompt_ids) + + @property + def num_prompt_tokens(self) -> int: + return len(self.prompt_ids) + + @property + def num_tokens(self) -> int: + return len(self.token_ids) + + @property + def num_completion_tokens(self) -> int: + return self.num_tokens - self.num_prompt_tokens + + @property + def completion_token_ids(self) -> List[List[int]]: + return self.token_ids[self.num_prompt_tokens:] + + @property + def last_token(self) -> List[int]: + return self.token_ids[-1] if self.token_ids else [] + + @property + def is_finished(self) -> bool: + """Check if sequence is finished. + + A sequence is finished when: + 1. EOS has been detected (eos_frame is set) + 2. The countdown has reached 0 + 3. Or max_tokens limit is reached + """ + if self.num_completion_tokens >= self.sampling_params.max_tokens: + return True + if self.sampling_params.ignore_eos: + return False + return self.eos_frame is not None and self.eos_countdown <= 0 + + def append_token(self, token: List[int]) -> None: + """Append a token and check for EOS.""" + self.token_ids.append(token) + self._check_eos(token) + + def _check_eos(self, token: List[int]) -> None: + """Check if EOS is detected in the delayed audio codebooks.""" + if self.eos_frame is None: + eos_cols = [token[i] == self.eoa_id for i in range(self.n_codebooks)] + if any(eos_cols): + step = self.num_completion_tokens - 1 + max_eos_cb = max(i for i, is_eos in enumerate(eos_cols) if is_eos) + self.eos_frame = max(0, step - max_eos_cb) + self.eos_countdown = self.n_codebooks + 1 + + if self.eos_frame is not None and self.eos_countdown > 0: + self.eos_countdown -= 1 + + def __len__(self) -> int: + return len(self.token_ids) + + +_seq_counter = 0 + + +def create_sequence( + prompt_ids: List[List[int]], + sampling_params: TTSSamplingParams, + n_codebooks: int = 9, + eoa_id: int = 1024, +) -> TTSSequence: + """Factory function to create a new TTS sequence with unique ID.""" + global _seq_counter + seq = TTSSequence( + prompt_ids=prompt_ids, + sampling_params=sampling_params, + seq_id=_seq_counter, + n_codebooks=n_codebooks, + eoa_id=eoa_id, + ) + _seq_counter += 1 + return seq + + +def reset_seq_counter() -> None: + """Reset the sequence counter (useful for testing).""" + global _seq_counter + _seq_counter = 0 diff --git a/zonos2/utils/__init__.py b/zonos2/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3dd400a32e404eb3cca89e9ba35774dc6d83b698 --- /dev/null +++ b/zonos2/utils/__init__.py @@ -0,0 +1,39 @@ +from .arch import is_arch_supported, is_sm90_supported, is_sm100_supported +from .hf import cached_load_checkpoint_config, resolve_model_path +from .logger import init_logger, set_log_level +from .misc import UNSET, Unset, call_if_main, divide_down, divide_even, divide_up +from .mp import ( + ZmqAsyncPullQueue, + ZmqAsyncPushQueue, + ZmqPubQueue, + ZmqPullQueue, + ZmqPushQueue, + ZmqSubQueue, +) +from .registry import Registry +from .torch_utils import nvtx_annotate, torch_dtype + +__all__ = [ + "cached_load_checkpoint_config", + "resolve_model_path", + "init_logger", + "set_log_level", + "is_arch_supported", + "is_sm90_supported", + "is_sm100_supported", + "call_if_main", + "divide_even", + "divide_up", + "divide_down", + "UNSET", + "Unset", + "torch_dtype", + "nvtx_annotate", + "Registry", + "ZmqPushQueue", + "ZmqPullQueue", + "ZmqPubQueue", + "ZmqSubQueue", + "ZmqAsyncPushQueue", + "ZmqAsyncPullQueue", +] diff --git a/zonos2/utils/arch.py b/zonos2/utils/arch.py new file mode 100644 index 0000000000000000000000000000000000000000..f30f50916c22a7f44a2190659903a25642d999f7 --- /dev/null +++ b/zonos2/utils/arch.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import Tuple + + +@lru_cache(maxsize=None) +def _get_torch_cuda_version() -> Tuple[int, int] | None: + import torch + import torch.version + + if not torch.cuda.is_available() or not torch.version.cuda: + return None + return torch.cuda.get_device_capability() + + +def is_arch_supported(major: int, minor: int = 0) -> bool: + arch = _get_torch_cuda_version() + if arch is None: + return False + return arch >= (major, minor) + + +def is_sm90_supported() -> bool: + return is_arch_supported(9, 0) + + +def is_sm100_supported() -> bool: + return is_arch_supported(10, 0) diff --git a/zonos2/utils/hf.py b/zonos2/utils/hf.py new file mode 100644 index 0000000000000000000000000000000000000000..6e62de00c549341341105874159915b4f8167b1f --- /dev/null +++ b/zonos2/utils/hf.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Optional + +from zonos2.utils.logger import init_logger +from zonos2.utils.moe_topk import normalize_special_topk_layers + +logger = init_logger(__name__) + +_HF_REPO_ID_RE = re.compile(r"^[\w.-]+/[\w.-]+$") + + +@lru_cache() +def resolve_model_path(model_path: str) -> str: + """Resolve a model path to a local directory. + + Local paths are returned as-is. A Hugging Face repo id (e.g. + ``Zyphra/ZONOS2``) is downloaded to the local HF cache first. + """ + if Path(model_path).expanduser().exists(): + return model_path + if _HF_REPO_ID_RE.match(model_path): + from huggingface_hub import snapshot_download + + logger.info("Downloading checkpoint from Hugging Face: %s", model_path) + return snapshot_download( + model_path, + allow_patterns=["*.json", "*.pth", "*.pt", "*.yaml"], + ) + return model_path + + +@dataclass +class Zonos2Config: + """Config class for zonos2 models loaded from training checkpoints. + + This mimics the HuggingFace config interface for compatibility with the server. + """ + + model_type: str = "zonos2" + dtype: str = "bfloat16" + + # Model architecture + n_layers: int = 8 + dim: int = 512 + head_dim: int = 128 + n_heads: Optional[int] = None + n_kv_heads: Optional[int] = None + ffn_dim_multiplier: float = 4.0 + multiple_of: int = 256 + norm_eps: float = 1e-5 + rope_theta: float = 10000.0 + max_seqlen: int = 2048 + + # TTS-specific + n_codebooks: int = 9 + codebook_size: int = 1024 + eoa_id: int = 1024 + audio_pad_id: int = 1025 + text_vocab: Optional[int] = None + speaker_enabled: bool = False + speaker_embedding_dim: int = 128 + # Optional LDA projection applied to speaker embeddings before the speaker + # projection. The projection weights live inside the model checkpoint. + speaker_lda_dim: Optional[int] = None + # Clean/noisy speaker-background marker tokens (2 ids at the text-vocab tail). + speaker_background_token_enabled: bool = False + # Accurate-mode marker token (1 id after the background markers). + accurate_mode_token_enabled: bool = False + speaking_rate_num_buckets: int = 0 + speaking_rate_buckets: Optional[list[str]] = None + quality_num_buckets: int = 0 + quality_features: Optional[list[str]] = None + quality_buckets: Optional[dict[str, list[str]]] = None + quality_dropout: Optional[dict[str, float]] = None + + # MoE config + moe_router_topk: int = 1 + special_topk_layers: dict[int, int] | None = None + moe_n_experts: int = 1 + moe_router_dim: int = 128 + moe_start_from_layer: int = 0 + moe_end_from_layer: int = 0 + moe_impl: str = "grouped" + moe_balancing_strategy: str = "legacy" + + # Loss + loss_softcap: float = 15.0 + + def __post_init__(self) -> None: + self.special_topk_layers = normalize_special_topk_layers( + self.special_topk_layers + ) + if self.speaking_rate_buckets is not None: + self.speaking_rate_buckets = [str(item) for item in self.speaking_rate_buckets] + if self.quality_features is not None: + self.quality_features = [str(item) for item in self.quality_features] + if self.quality_buckets is not None: + self.quality_buckets = { + str(feature): [str(item) for item in (buckets or [])] + for feature, buckets in self.quality_buckets.items() + } + if self.quality_features is None: + self.quality_features = list(self.quality_buckets.keys()) + if int(self.quality_num_buckets or 0) <= 0: + self.quality_num_buckets = sum( + len(self.quality_buckets.get(feature, ())) + for feature in (self.quality_features or []) + ) + if self.quality_dropout is not None: + self.quality_dropout = { + str(feature): float(dropout) + for feature, dropout in self.quality_dropout.items() + } + self.speaker_background_token_enabled = bool(self.speaker_background_token_enabled) + self.accurate_mode_token_enabled = bool(self.accurate_mode_token_enabled) + + def to_dict(self): + return self.__dict__.copy() + + +def _load_zonos2_config(model_path: str) -> Zonos2Config: + """Load config from zonos2 training checkpoint format. + + Supports: + - config.yaml in parent directory (training run format) + - params.json in checkpoint directory (release checkpoint format) + """ + from pathlib import Path + + def _cfg_get(cfg, key: str, default=None): + if cfg is None: + return default + if isinstance(cfg, dict): + return cfg.get(key, default) + return getattr(cfg, key, default) + + def _apply_data_sidecar(model_params: dict, data_cfg) -> None: + if data_cfg is None: + return + + rate_buckets = _cfg_get(data_cfg, "speaking_rate_buckets", None) or [] + rate_buckets = [str(item) for item in rate_buckets] + if rate_buckets: + model_params["speaking_rate_buckets"] = rate_buckets + + rate_count = int(model_params.get("speaking_rate_num_buckets") or 0) + rate_enabled = bool(_cfg_get(data_cfg, "speaking_rate_enabled", False)) + if rate_enabled: + sidecar_rate_count = len(rate_buckets) + if sidecar_rate_count == 0: + sidecar_rate_count = int(_cfg_get(data_cfg, "speaking_rate_num_buckets", 0) or 0) + if sidecar_rate_count > 0: + rate_count = sidecar_rate_count + if not int(model_params.get("speaking_rate_num_buckets") or 0): + model_params["speaking_rate_num_buckets"] = rate_count + + # Quality conditioning lives in the data section of training configs. + if bool(_cfg_get(data_cfg, "quality_enabled", False)): + raw_features = _cfg_get(data_cfg, "quality_features", None) + if hasattr(raw_features, "items"): + quality_features = [ + str(feature) for feature, enabled in raw_features.items() if bool(enabled) + ] + else: + quality_features = [str(item) for item in (raw_features or ())] + raw_buckets = _cfg_get(data_cfg, "quality_buckets", None) or {} + quality_buckets = { + str(feature): [str(item) for item in (raw_buckets.get(feature, None) or ())] + for feature in (quality_features or raw_buckets.keys()) + } + if quality_buckets and "quality_buckets" not in model_params: + model_params["quality_buckets"] = quality_buckets + model_params["quality_features"] = quality_features or list( + quality_buckets.keys() + ) + raw_dropout = _cfg_get(data_cfg, "quality_dropout", None) + if raw_dropout is not None and "quality_dropout" not in model_params: + if hasattr(raw_dropout, "items"): + model_params["quality_dropout"] = { + str(feature): float(dropout) for feature, dropout in raw_dropout.items() + } + + # Training configs name the marker-token flags after their data pipeline. + background_enabled = _cfg_get(data_cfg, "speaker_embedding_origin_token_enabled", None) + if background_enabled is not None: + model_params.setdefault( + "speaker_background_token_enabled", bool(background_enabled) + ) + accurate_enabled = _cfg_get( + data_cfg, "speaker_embedding_cartesia_clone_source_token_enabled", None + ) + if accurate_enabled is not None: + model_params.setdefault("accurate_mode_token_enabled", bool(accurate_enabled)) + + if rate_count > 0 and model_params.get("text_vocab") is None: + try: + from zonos2.tts.prompt import conditioned_text_vocab_size + + quality_count = sum( + len(buckets) + for buckets in (model_params.get("quality_buckets") or {}).values() + ) + background_count = ( + 2 if model_params.get("speaker_background_token_enabled") else 0 + ) + accurate_count = ( + 1 + if model_params.get("accurate_mode_token_enabled") and background_count + else 0 + ) + model_params["text_vocab"] = conditioned_text_vocab_size( + rate_count, quality_count, background_count, accurate_count + ) + logger.debug( + "Resolved text_vocab=%d from data text vocabulary and conditioning buckets", + model_params["text_vocab"], + ) + except Exception as exc: + logger.debug("Could not resolve conditioned text_vocab from data config: %s", exc) + + def _validate_model_type(model_params: dict) -> None: + model_type = model_params.get("model_type") + if model_type is not None and str(model_type) != "zonos2": + raise ValueError( + f"Unsupported model_type={model_type!r}. This release only loads zonos2 checkpoints." + ) + + model_path = Path(model_path) + + # Try params.json in checkpoint dir first + params_json = model_path / "params.json" + if params_json.exists(): + with open(params_json, "r") as f: + params = json.load(f) + # params.json may have nested structure + if "model" in params: + model_params = params["model"] + else: + model_params = params + + # Also check config.yaml for tokenizer sidecar fields stored in data. + for parent in [model_path, model_path.parent, model_path.parent.parent]: + config_yaml = parent / "config.yaml" + if config_yaml.exists(): + try: + from omegaconf import OmegaConf + + cfg = OmegaConf.load(config_yaml) + _apply_data_sidecar(model_params, getattr(cfg, "data", None)) + except ImportError: + import yaml + + with open(config_yaml, "r") as f: + cfg = yaml.safe_load(f) + _apply_data_sidecar( + model_params, cfg.get("data") if isinstance(cfg, dict) else None + ) + break + + _validate_model_type(model_params) + result = Zonos2Config( + **{k: v for k, v in model_params.items() if hasattr(Zonos2Config, k)} + ) + logger.debug("Loaded Zonos2Config from params.json") + return result + + # Try config.yaml in parent directories + for parent in [model_path, model_path.parent, model_path.parent.parent]: + config_yaml = parent / "config.yaml" + logger.debug("Checking for config at: %s", config_yaml) + if config_yaml.exists(): + logger.debug("Found config.yaml at: %s", config_yaml) + try: + from omegaconf import OmegaConf + + cfg = OmegaConf.load(config_yaml) + # Navigate to model config + if hasattr(cfg, "model"): + model_cfg = OmegaConf.to_container(cfg.model, resolve=True) + else: + model_cfg = OmegaConf.to_container(cfg, resolve=True) + # Also check data section for tokenizer sidecar fields. + _apply_data_sidecar(model_cfg, getattr(cfg, "data", None)) + _validate_model_type(model_cfg) + result = Zonos2Config( + **{k: v for k, v in model_cfg.items() if hasattr(Zonos2Config, k)} + ) + logger.debug("Loaded Zonos2Config from config.yaml") + return result + except ImportError: + # omegaconf not available, try as regular yaml + import yaml + + with open(config_yaml, "r") as f: + cfg = yaml.safe_load(f) + model_cfg = cfg.get("model", cfg) + # Also check data section for tokenizer sidecar fields. + _apply_data_sidecar(model_cfg, cfg.get("data") if isinstance(cfg, dict) else None) + _validate_model_type(model_cfg) + result = Zonos2Config( + **{k: v for k, v in model_cfg.items() if hasattr(Zonos2Config, k)} + ) + logger.debug("Loaded Zonos2Config from config.yaml (yaml)") + return result + + raise ValueError( + f"Could not find config.yaml or params.json for zonos2 checkpoint at {model_path}" + ) + + +def _is_zonos2_checkpoint(model_path: str) -> bool: + """Check if path is a zonos2 checkpoint with a loadable config.""" + model_path = Path(model_path) + + # Check for params.json + if (model_path / "params.json").exists(): + return True + + # Check for config.yaml in parent dirs (training run format) + for parent in [model_path, model_path.parent, model_path.parent.parent]: + if (parent / "config.yaml").exists(): + return True + + return False + + +@lru_cache() +def _load_config(model_path: str) -> Zonos2Config: + model_path = resolve_model_path(model_path) + if not _is_zonos2_checkpoint(model_path): + raise ValueError( + f"Unsupported checkpoint at {model_path!r}. This release only loads " + "Zonos2 TTS checkpoints with config.yaml or params.json." + ) + return _load_zonos2_config(model_path) + + +def cached_load_checkpoint_config(model_path: str) -> Zonos2Config: + config = _load_config(model_path) + return Zonos2Config(**config.to_dict()) diff --git a/zonos2/utils/logger.py b/zonos2/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..f61815da0b3e38bce411bb8238cc0d2779b252c9 --- /dev/null +++ b/zonos2/utils/logger.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from functools import partial +from typing import TYPE_CHECKING + +_LOG_LEVEL = None + +_LEVEL_MAP = { + "DEBUG": 10, # logging.DEBUG + "INFO": 20, # logging.INFO + "WARNING": 30, # logging.WARNING + "ERROR": 40, # logging.ERROR + "CRITICAL": 50, # logging.CRITICAL +} + + +def set_log_level(level: int | str) -> None: + """Set the global log level for all zonos2 loggers. + + Args: + level: Either a logging level int (e.g. logging.DEBUG) or a string + like "DEBUG", "INFO", etc. + """ + import logging + + global _LOG_LEVEL + if isinstance(level, str): + level = _LEVEL_MAP.get(level.upper(), logging.INFO) + _LOG_LEVEL = level + + # Update all existing loggers under the zonos2 namespace + for name, obj in logging.Logger.manager.loggerDict.items(): + if isinstance(obj, logging.Logger) and name.startswith("zonos2"): + obj.setLevel(level) + + +def init_logger( + name: str, + suffix: str = "", + *, + strip_file: bool = True, + level: str | None = None, + use_pid: bool | None = None, + use_tp_rank: bool | None = None, +): + """Initialize the logger for the module with colors and pretty formatting.""" + import logging + import os + import sys + + global _LOG_LEVEL + if _LOG_LEVEL is None: + level = level or os.getenv("LOG_LEVEL", "").upper() + _LOG_LEVEL = _LEVEL_MAP.get(level, logging.INFO) + + if strip_file: + suffix = os.path.basename(suffix) + + if suffix: + suffix = f"|{suffix}" + + if use_pid is None: + use_pid = os.getenv("LOG_PID", "0").lower() in ("1", "true", "yes") + + if use_pid: + pid = os.getpid() + suffix = f"|pid={pid}{suffix}" + + tp_info = None + + # Color formatter class + class ColorFormatter(logging.Formatter): + """Formatter with colors and pretty output""" + + # ANSI color codes + COLORS = { + "DEBUG": "\033[36m", # Cyan + "INFO": "\033[32m", # Green + "WARNING": "\033[33m", # Yellow + "ERROR": "\033[31m", # Red + "CRITICAL": "\033[35m", # Magenta + } + RESET = "\033[0m" + BOLD = "\033[1m" + + def format(self, record): + from zonos2.distributed import try_get_tp_info + + # Format timestamp like SGLang: [YYYY-MM-DD|HH:MM:SS|pid=1234] + timestamp = self.formatTime(record, "[%Y-%m-%d|%H:%M:%S{suffix}]") + nonlocal tp_info + tp_info = tp_info or try_get_tp_info() + if tp_info is not None and use_tp_rank is not False: + real_suffix = f"{suffix}|core|rank={tp_info.rank}" + else: + real_suffix = suffix + timestamp = timestamp.format(suffix=real_suffix) + + # Get color for log level + level_color = self.COLORS.get(record.levelname, "") + + # Format the message + colored_level = f"{level_color}{record.levelname:<8}{self.RESET}" + message = record.getMessage() + + # Pretty format: [timestamp] LEVEL message + return f"{self.BOLD}{timestamp}{self.RESET} {colored_level} {message}" + + logger = logging.getLogger(name) + logger.setLevel(_LOG_LEVEL) + + # Clear existing handlers to avoid duplicates + logger.handlers.clear() + + handler = logging.StreamHandler(sys.stdout) + formatter = ColorFormatter() + handler.setFormatter(formatter) + logger.addHandler(handler) + + # Prevent propagation to root logger + logger.propagate = False + + def _call_rank0(msg, *args, _which, **kwargs): + from zonos2.distributed import get_tp_info + + nonlocal tp_info + tp_info = tp_info or get_tp_info() + assert tp_info is not None, "TP info not set yet" + if tp_info.is_primary(): + getattr(logger, _which)(msg, *args, **kwargs) + + if TYPE_CHECKING: + + class WrapperLogger(logging.Logger): + """Custom logger to handle the color formatter.""" + + def info_rank0(self, msg, *args, **kwargs): ... + def warning_rank0(self, msg, *args, **kwargs): ... + def debug_rank0(self, msg, *args, **kwargs): ... + def critical_rank0(self, msg, *args, **kwargs): ... + + return WrapperLogger(name) + else: + logger.info_rank0 = partial(_call_rank0, _which="info") + logger.debug_rank0 = partial(_call_rank0, _which="debug") + logger.critical_rank0 = partial(_call_rank0, _which="critical") + logger.warning_rank0 = partial(_call_rank0, _which="warning") + return logger diff --git a/zonos2/utils/misc.py b/zonos2/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..d7313ba053aea8ac65549a1feb2c9d7fb3950a9f --- /dev/null +++ b/zonos2/utils/misc.py @@ -0,0 +1,40 @@ +from __future__ import annotations + + +def call_if_main(name: str = "__main__", discard: bool | None = None): + """Decorator to ensure a function will call when the script is run as main.""" + if name != "__main__": + discard = False if discard is None else discard + if discard: + return lambda _: None + else: + return lambda f: f + else: + discard = True if discard is None else discard + if discard: + return lambda f: (f() or True) and None + else: + return lambda f: (f() and None) or f + + +def divide_even(a: int, b: int) -> int: + """Divides two integers""" + assert a % b == 0, f"{a = } must be divisible by {b = }" + return a // b + + +def divide_up(a: int, b: int) -> int: + """Divides two integers, rounding up""" + return (a + b - 1) // b + + +def divide_down(a: int, b: int) -> int: + """Divides two integers, rounding down""" + return a // b + + +class Unset: + pass + + +UNSET = Unset() diff --git a/zonos2/utils/moe_topk.py b/zonos2/utils/moe_topk.py new file mode 100644 index 0000000000000000000000000000000000000000..5e02740b51b32a70e6a77ba63739d7c8a72af53f --- /dev/null +++ b/zonos2/utils/moe_topk.py @@ -0,0 +1,37 @@ +from collections.abc import Mapping +from typing import Any + + +def normalize_special_topk_layers( + special_topk_layers: Mapping[Any, Any] | None, +) -> dict[int, int] | None: + if special_topk_layers is None: + return None + + normalized: dict[int, int] = {} + for layer_idx, topk in special_topk_layers.items(): + layer_idx = int(layer_idx) + topk = int(topk) + if topk < 1: + raise ValueError( + f"special_topk_layers[{layer_idx}] must be >= 1, got {topk}" + ) + normalized[layer_idx] = topk + return normalized + + +def resolve_layer_topk( + default_topk: int, + special_topk_layers: Mapping[Any, Any] | None, + layer_idx: int, +) -> int: + topk = default_topk + if special_topk_layers is not None: + topk = special_topk_layers.get( + layer_idx, special_topk_layers.get(str(layer_idx), default_topk) + ) + + topk = int(topk) + if topk < 1: + raise ValueError(f"top-k for layer {layer_idx} must be >= 1, got {topk}") + return topk diff --git a/zonos2/utils/mp.py b/zonos2/utils/mp.py new file mode 100644 index 0000000000000000000000000000000000000000..181baf1a07fcbbc75c9056ae4f632426134e1c20 --- /dev/null +++ b/zonos2/utils/mp.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from typing import Callable, Dict, Generic, TypeVar + +import msgpack +import zmq +import zmq.asyncio + +T = TypeVar("T") + + +class ZmqPushQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + encoder: Callable[[T], Dict], + ): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.PUSH) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.encoder = encoder + + def put(self, obj: T): + event = msgpack.packb(self.encoder(obj), use_bin_type=True) + self.socket.send(event, copy=False) + + def stop(self): + self.socket.close() + self.context.term() + + +class ZmqAsyncPushQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + encoder: Callable[[T], Dict], + ): + self.context = zmq.asyncio.Context() + self.socket = self.context.socket(zmq.PUSH) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.encoder = encoder + + async def put(self, obj: T): + event = msgpack.packb(self.encoder(obj), use_bin_type=True) + await self.socket.send(event, copy=False) + + def stop(self): + self.socket.close() + self.context.term() + + +class ZmqPullQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + decoder: Callable[[Dict], T], + ): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.PULL) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.decoder = decoder + + def get(self) -> T: + event = self.socket.recv() + return self.decoder(msgpack.unpackb(event, raw=False)) + + def get_raw(self) -> bytes: + return self.socket.recv() + + def decode(self, raw: bytes) -> T: + return self.decoder(msgpack.unpackb(raw, raw=False)) + + def empty(self) -> bool: + return self.socket.poll(timeout=0) == 0 + + def stop(self): + self.socket.close() + self.context.term() + + +class ZmqAsyncPullQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + decoder: Callable[[Dict], T], + ): + self.context = zmq.asyncio.Context() + self.socket = self.context.socket(zmq.PULL) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.decoder = decoder + + async def get(self) -> T: + event = await self.socket.recv() + return self.decoder(msgpack.unpackb(event, raw=False)) + + def stop(self): + self.socket.close() + self.context.term() + + +class ZmqPubQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + encoder: Callable[[T], Dict], + ): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.PUB) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.encoder = encoder + + def put_raw(self, raw: bytes): + self.socket.send(raw, copy=False) + + def put(self, obj: T): + event = msgpack.packb(self.encoder(obj), use_bin_type=True) + self.socket.send(event, copy=False) + + def stop(self): + self.socket.close() + self.context.term() + + +class ZmqSubQueue(Generic[T]): + def __init__( + self, + addr: str, + create: bool, + decoder: Callable[[Dict], T], + ): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.SUB) + self.socket.bind(addr) if create else self.socket.connect(addr) + self.socket.setsockopt_string(zmq.SUBSCRIBE, "") + self.decoder = decoder + + def get(self) -> T: + event = self.socket.recv() + return self.decoder(msgpack.unpackb(event, raw=False)) + + def empty(self) -> bool: + return self.socket.poll(timeout=0) == 0 + + def stop(self): + self.socket.close() + self.context.term() diff --git a/zonos2/utils/registry.py b/zonos2/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..4c54a9a145f36c6a359d3c30a933d1f9ceb1dfd1 --- /dev/null +++ b/zonos2/utils/registry.py @@ -0,0 +1,26 @@ +from typing import Callable, Generic, List, TypeVar + +T = TypeVar("T") + + +class Registry(Generic[T]): + def __init__(self, type: str): + self._registry = {} + self._type = type + + def register(self, name: str) -> Callable[[T], None]: + if name in self._registry: + raise KeyError(f"{self._type} '{name}' is already registered.") + + def decorator(item: T) -> None: + self._registry[name] = item + + return decorator + + def __getitem__(self, name: str) -> T: + if name not in self._registry: + raise KeyError(f"Unsupported {self._type}: {name}") + return self._registry[name] + + def supported_names(self) -> List[str]: + return list(self._registry.keys()) diff --git a/zonos2/utils/torch_utils.py b/zonos2/utils/torch_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9422b9e7de187b2cecfa49db83575bd492faba77 --- /dev/null +++ b/zonos2/utils/torch_utils.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import functools +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import torch + + +@contextmanager +def torch_dtype(dtype: torch.dtype): + import torch # real import when used + + old_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(old_dtype) + + +def nvtx_annotate(name: str, layer_id_field: str | None = None): + import torch.cuda.nvtx as nvtx + + def decorator(fn): + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + display_name = name + if layer_id_field and hasattr(self, layer_id_field): + display_name = name.format(getattr(self, layer_id_field)) + with nvtx.range(display_name): + return fn(self, *args, **kwargs) + + return wrapper + + return decorator diff --git a/zonos2/vendor/__init__.py b/zonos2/vendor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ad4f5f092f22219192c9f48320a2acd5e6baa0c6 --- /dev/null +++ b/zonos2/vendor/__init__.py @@ -0,0 +1 @@ +"""Vendored third-party code. See tools/vendor_nemo_tn.py.""" diff --git a/zonos2/vendor/nemo_text_processing/__init__.py b/zonos2/vendor/nemo_text_processing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/package_info.py b/zonos2/vendor/nemo_text_processing/package_info.py new file mode 100644 index 0000000000000000000000000000000000000000..c51954122f14d009fa39356f441b351dfbc65efc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/package_info.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MAJOR = 1 +MINOR = 2 +PATCH = 0 +PRE_RELEASE = '' + +# Use the following formatting: (major, minor, patch, pre-release) +VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE) + +__shortversion__ = '.'.join(map(str, VERSION[:3])) +__version__ = '.'.join(map(str, VERSION[:3])) + ''.join(VERSION[3:]) + +__package_name__ = 'nemo_text_processing' +__contact_names__ = 'NVIDIA' +__contact_emails__ = 'nemo-toolkit@nvidia.com' +__homepage__ = 'https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/' +__repository_url__ = 'https://github.com/nvidia/nemo-text-processing' +__download_url__ = 'https://github.com/NVIDIA/NeMo-text-processing/releases' +__description__ = 'NeMo text processing for ASR and TTS' +__license__ = 'Apache2' +__keywords__ = ' NeMo, nvidia, tts, asr, text processing, text normalization, inverse text normalization, language' diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..715f9c2908187b9fbc35dcdc8e8fe4cbb0fc6859 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.normalize import Normalizer diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/data_loader_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/data_loader_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..41733424804173af9e2db9dc86a251ccd4d5118a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/data_loader_utils.py @@ -0,0 +1,361 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import re +import string +import sys +import unicodedata +from collections import defaultdict, namedtuple +from typing import Dict, List, Optional, Set, Tuple +from unicodedata import category + +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +NFC = 'NFC' +EOS_TYPE = "EOS" +PUNCT_TYPE = "PUNCT" +PLAIN_TYPE = "PLAIN" +Instance = namedtuple('Instance', 'token_type un_normalized normalized') +known_types = [ + "PLAIN", + "DATE", + "CARDINAL", + "LETTERS", + "VERBATIM", + "MEASURE", + "DECIMAL", + "ORDINAL", + "DIGIT", + "MONEY", + "TELEPHONE", + "ELECTRONIC", + "FRACTION", + "TIME", + "ADDRESS", + "ROMAN", + "RANGE", +] + + +def _load_kaggle_text_norm_file(file_path: str, to_lower: bool) -> List[Instance]: + """ + https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish + Loads text file in the Kaggle Google text normalization file format: \t\t<`self` if trivial class or normalized text> + E.g. + PLAIN Brillantaisia + PLAIN is + PLAIN a + PLAIN genus + PLAIN of + PLAIN plant + PLAIN in + PLAIN family + PLAIN Acanthaceae + PUNCT . sil + + + Args: + file_path: file path to text file + + Returns: flat list of instances + """ + res = [] + with open(file_path, 'r') as fp: + for line in fp: + parts = line.strip().split("\t") + if parts[0] == "": + res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized="")) + else: + l_type, l_token, l_normalized = parts + if to_lower: + l_token = l_token.lower() + l_normalized = l_normalized.lower() + + if l_type == PLAIN_TYPE: + res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_token)) + elif l_type != PUNCT_TYPE: + res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_normalized)) + return res + + +def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file, to_lower: bool = True) -> List[Instance]: + """ + Load given list of text files using the `load_func` function. + + Args: + file_paths: list of file paths + load_func: loading function + + Returns: flat list of instances + """ + res = [] + for file_path in file_paths: + res.extend(load_func(file_path=file_path, to_lower=to_lower)) + return res + + +def clean_generic(text: str) -> str: + """ + Cleans text without affecting semiotic classes. + + Args: + text: string + + Returns: cleaned string + """ + text = text.strip() + text = text.lower() + return text + + +def evaluate(preds: List[str], labels: List[str], input: Optional[List[str]] = None, verbose: bool = True) -> float: + """ + Evaluates accuracy given predictions and labels. + + Args: + preds: predictions + labels: labels + input: optional, only needed for verbosity + verbose: if true prints [input], golden labels and predictions + + Returns accuracy + """ + acc = 0 + nums = len(preds) + for i in range(nums): + pred_norm = clean_generic(preds[i]) + label_norm = clean_generic(labels[i]) + if pred_norm == label_norm: + acc = acc + 1 + else: + if input: + print(f"input: {json.dumps(input[i])}") + print(f"gold: {json.dumps(label_norm)}") + print(f"pred: {json.dumps(pred_norm)}") + return acc / nums + + +def training_data_to_tokens( + data: List[Instance], category: Optional[str] = None +) -> Dict[str, Tuple[List[str], List[str]]]: + """ + Filters the instance list by category if provided and converts it into a map from token type to list of un_normalized and normalized strings + + Args: + data: list of instances + category: optional semiotic class category name + + Returns Dict: token type -> (list of un_normalized strings, list of normalized strings) + """ + result = defaultdict(lambda: ([], [])) + for instance in data: + if instance.token_type != EOS_TYPE: + if category is None or instance.token_type == category: + result[instance.token_type][0].append(unicodedata.normalize(NFC, instance.un_normalized)) + result[instance.token_type][1].append(unicodedata.normalize(NFC, instance.normalized)) + return result + + +def training_data_to_sentences(data: List[Instance]) -> Tuple[List[str], List[str], List[Set[str]]]: + """ + Takes instance list, creates list of sentences split by EOS_Token + Args: + data: list of instances + Returns (list of unnormalized sentences, list of normalized sentences, list of sets of categories in a sentence) + """ + # split data at EOS boundaries + sentences = [] + sentence = [] + categories = [] + sentence_categories = set() + + for instance in data: + if instance.token_type == EOS_TYPE: + sentences.append(sentence) + sentence = [] + categories.append(sentence_categories) + sentence_categories = set() + else: + sentence.append(instance) + sentence_categories.update([instance.token_type]) + un_normalized = [ + " ".join([unicodedata.normalize(NFC, instance.un_normalized) for instance in sentence]) + for sentence in sentences + ] + normalized = [ + " ".join([unicodedata.normalize(NFC, instance.normalized) for instance in sentence]) for sentence in sentences + ] + return un_normalized, normalized, categories + + +def post_process_punctuation(text: str) -> str: + """ + Normalized quotes and spaces + + Args: + text: text + + Returns: text with normalized spaces and quotes + """ + text = ( + text.replace('( ', '(') + .replace(' )', ')') + .replace('{ ', '{') + .replace(' }', '}') + .replace('[ ', '[') + .replace(' ]', ']') + .replace(' ', ' ') + .replace('”', '"') + .replace("’", "'") + .replace("»", '"') + .replace("«", '"') + .replace("\\", "") + .replace("„", '"') + .replace("´", "'") + .replace("’", "'") + .replace('“', '"') + .replace("‘", "'") + .replace('`', "'") + .replace('- -', "--") + ) + + for punct in "!,.:;?": + text = text.replace(f' {punct}', punct) + return text.strip() + + +def pre_process(text: str) -> str: + """ + Optional text preprocessing before normalization (part of TTS TN pipeline) + + Args: + text: string that may include semiotic classes + + Returns: text with spaces around punctuation marks + """ + space_both = '[]' + for punct in space_both: + text = text.replace(punct, ' ' + punct + ' ') + + # remove extra space + text = re.sub(r' +', ' ', text) + return text + + +def load_file(file_path: str) -> List[str]: + """ + Loads given text file with separate lines into list of string. + + Args: + file_path: file path + + Returns: flat list of string + """ + res = [] + with open(file_path, 'r') as fp: + for line in fp: + res.append(line) + return res + + +def write_file(file_path: str, data: List[str]): + """ + Writes out list of string to file. + + Args: + file_path: file path + data: list of string + + """ + with open(file_path, 'w') as fp: + for line in data: + fp.write(line + '\n') + + +def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool = False): + """ + Post-processing of the normalized output to match input in terms of spaces around punctuation marks. + After NN normalization, Moses detokenization puts a space after + punctuation marks, and attaches an opening quote "'" to the word to the right. + E.g., input to the TN NN model is "12 test' example", + after normalization and detokenization -> "twelve test 'example" (the quote is considered to be an opening quote, + but it doesn't match the input and can cause issues during TTS voice generation.) + The current function will match the punctuation and spaces of the normalized text with the input sequence. + "12 test' example" -> "twelve test 'example" -> "twelve test' example" (the quote was shifted to match the input). + + Args: + input: input text (original input to the NN, before normalization or tokenization) + normalized_text: output text (output of the TN NN model) + add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time) + """ + # in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly) + # this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement + # to make sure these new double quotes work with this function + if "``" in input and "``" not in normalized_text: + input = input.replace("``", '"') + input = [x for x in input] + normalized_text = [x for x in normalized_text] + punct_marks = [x for x in string.punctuation if x in input] + + if add_unicode_punct: + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_marks and chr(i) in input + ] + punct_marks = punct_marks.extend(punct_unicode) + + for punct in punct_marks: + try: + equal = True + if input.count(punct) != normalized_text.count(punct): + equal = False + idx_in, idx_out = 0, 0 + while punct in input[idx_in:]: + idx_out = normalized_text.index(punct, idx_out) + idx_in = input.index(punct, idx_in) + + def _is_valid(idx_out, idx_in, normalized_text, input): + """Check if previous or next word match (for cases when punctuation marks are part of + semiotic token, i.e. some punctuation can be missing in the normalized text)""" + return (idx_out > 0 and idx_in > 0 and normalized_text[idx_out - 1] == input[idx_in - 1]) or ( + idx_out < len(normalized_text) - 1 + and idx_in < len(input) - 1 + and normalized_text[idx_out + 1] == input[idx_in + 1] + ) + + if not equal and not _is_valid(idx_out, idx_in, normalized_text, input): + idx_in += 1 + continue + if idx_in > 0 and idx_out > 0: + if normalized_text[idx_out - 1] == " " and input[idx_in - 1] != " ": + normalized_text[idx_out - 1] = "" + + elif normalized_text[idx_out - 1] != " " and input[idx_in - 1] == " ": + normalized_text[idx_out - 1] += " " + + if idx_in < len(input) - 1 and idx_out < len(normalized_text) - 1: + if normalized_text[idx_out + 1] == " " and input[idx_in + 1] != " ": + normalized_text[idx_out + 1] = "" + elif normalized_text[idx_out + 1] != " " and input[idx_in + 1] == " ": + normalized_text[idx_out] = normalized_text[idx_out] + " " + idx_out += 1 + idx_in += 1 + except: + logger.info(f"Skipping post-processing of {''.join(normalized_text)} for '{punct}'") + + normalized_text = "".join(normalized_text) + return re.sub(r' +', ' ', normalized_text) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..771fc5049b1bf2c7dd0f53318ee496dcefbd10cf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv @@ -0,0 +1,844 @@ +.com punkt com +.net punkt net +.org punkt org +.jp punkt jp +.de punkt de +.uk punkt uk +.fr punkt fr +.br punkt br +.it punkt it +.ru punkt ru +.es punkt es +.me punkt me +.gov punkt gov +.pl punkt pl +.ca punkt ca +.au punkt au +.cn punkt cn +.co punkt co +.in punkt in +.nl punkt nl +.edu punkt edu +.info punkt info +.eu punkt eu +.ch punkt ch +.id punkt id +.at punkt at +.kr punkt kr +.cz punkt cz +.mx punkt mx +.be punkt be +.tv punkt tv +.se punkt se +.tr punkt tr +.tw punkt tw +.al punkt al +.ua punkt ua +.ir punkt ir +.vn punkt vn +.cl punkt cl +.sk punkt sk +.ly punkt ly +.cc punkt cc +.to punkt to +.no punkt no +.fi punkt fi +.us punkt us +.pt punkt pt +.dk punkt dk +.ar punkt ar +.hu punkt hu +.tk punkt tk +.gr punkt gr +.il punkt il +.news punkt news +.ro punkt ro +.my punkt my +.biz punkt biz +.ie punkt ie +.za punkt za +.nz punkt nz +.sg punkt sg +.ee punkt ee +.th punkt th +.io punkt io +.xyz punkt xyz +.pe punkt pe +.bg punkt bg +.hk punkt hk +.rs punkt rs +.lt punkt lt +.link punkt link +.ph punkt ph +.club punkt club +.si punkt si +.site punkt site +.mobi punkt mobi +.by punkt by +.cat punkt cat +.wiki punkt wiki +.la punkt la +.ga punkt ga +.xxx punkt xxx +.cf punkt cf +.hr punkt hr +.ng punkt ng +.jobs punkt jobs +.online punkt online +.kz punkt kz +.ug punkt ug +.gq punkt gq +.ae punkt ae +.is punkt is +.lv punkt lv +.pro punkt pro +.fm punkt fm +.tips punkt tips +.ms punkt ms +.sa punkt sa +.app punkt app +.lat punkt lat +.pk punkt pk +.ws punkt ws +.top punkt top +.xn--p1ai punkt xn--p1ai +.pw punkt pw +.ai punkt ai +.kw punkt kw +.ml punkt ml +.su punkt su +.lu punkt lu +.nu punkt nu +.ec punkt ec +.uy punkt uy +.az punkt az +.ma punkt ma +.st punkt st +.asia punkt asia +.im punkt im +.am punkt am +.email punkt email +.ke punkt ke +.eg punkt eg +.live punkt live +.md punkt md +.uz punkt uz +.today punkt today +.ge punkt ge +.ba punkt ba +.bo punkt bo +.blog punkt blog +.lk punkt lk +.do punkt do +.one punkt one +.ve punkt ve +.media punkt media +.sh punkt sh +.vip punkt vip +.life punkt life +.guru punkt guru +.mp punkt mp +.int punkt int +.tz punkt tz +.desi punkt desi +.jo punkt jo +.mk punkt mk +.np punkt np +.py punkt py +.mu punkt mu +.mn punkt mn +.re punkt re +.travel punkt travel +.tn punkt tn +.gd punkt gd +.sn punkt sn +.cam punkt cam +.shop punkt shop +.mil punkt mil +.tokyo punkt tokyo +.gg punkt gg +.kg punkt kg +.cr punkt cr +.li punkt li +.cu punkt cu +.hn punkt hn +.eus punkt eus +.rocks punkt rocks +.bd punkt bd +.dev punkt dev +.fun punkt fun +.video punkt video +.gt punkt gt +.bz punkt bz +.space punkt space +.store punkt store +.mt punkt mt +.name punkt name +.work punkt work +.porn punkt porn +.pub punkt pub +.kh punkt kh +.icu punkt icu +.as punkt as +.cy punkt cy +.pr punkt pr +.careers punkt careers +.ps punkt ps +.dz punkt dz +.iq punkt iq +.ltd punkt ltd +.pa punkt pa +.sv punkt sv +.qa punkt qa +.design punkt design +.sc punkt sc +.sy punkt sy +.aero punkt aero +.tech punkt tech +.best punkt best +.scot punkt scot +.win punkt win +.ni punkt ni +.bzh punkt bzh +.lb punkt lb +.city punkt city +.digital punkt digital +.red punkt red +.pg punkt pg +.world punkt world +.ag punkt ag +.ci punkt ci +.report punkt report +.ac punkt ac +.center punkt center +.ad punkt ad +.ovh punkt ovh +.om punkt om +.cloud punkt cloud +.press punkt press +.zw punkt zw +.vg punkt vg +.cx punkt cx +.website punkt website +.plus punkt plus +.page punkt page +.bs punkt bs +.cd punkt cd +.gh punkt gh +.moe punkt moe +.help punkt help +.download punkt download +.ao punkt ao +.cm punkt cm +.af punkt af +.click punkt click +.mg punkt mg +.tm punkt tm +.na punkt na +.pics punkt pics +.mv punkt mv +.market punkt market +.rw punkt rw +.fo punkt fo +.fj punkt fj +.tl punkt tl +.gratis punkt gratis +.art punkt art +.mc punkt mc +.mo punkt mo +.gal punkt gal +.coop punkt coop +.nc punkt nc +.run punkt run +.mz punkt mz +.tj punkt tj +.zone punkt zone +.cv punkt cv +.bh punkt bh +.place punkt place +.gy punkt gy +.agency punkt agency +.kn punkt kn +.mm punkt mm +.bid punkt bid +.bj punkt bj +.je punkt je +.gs punkt gs +.global punkt global +.lol punkt lol +.tube punkt tube +.lc punkt lc +.bm punkt bm +.ninja punkt ninja +.bn punkt bn +.review punkt review +.bi punkt bi +.ht punkt ht +.sm punkt sm +.ax punkt ax +.pm punkt pm +.ink punkt ink +.vc punkt vc +.so punkt so +.tel punkt tel +.cw punkt cw +.uno punkt uno +.casa punkt casa +.network punkt network +.buzz punkt buzz +.mw punkt mw +.vu punkt vu +.guide punkt guide +.bf punkt bf +.mq punkt mq +.sex punkt sex +.sr punkt sr +.zm punkt zm +.onl punkt onl +.vet punkt vet +.bw punkt bw +.jm punkt jm +.bar punkt bar +.gp punkt gp +.pf punkt pf +.ne punkt ne +.va punkt va +.bt punkt bt +.sd punkt sd +.expert punkt expert +.kim punkt kim +.africa punkt africa +.mr punkt mr +.gi punkt gi +.tg punkt tg +.cg punkt cg +.ls punkt ls +.sb punkt sb +.ck punkt ck +.gf punkt gf +.yt punkt yt +.dm punkt dm +.realtor punkt realtor +.aw punkt aw +.sz punkt sz +.london punkt london +.et punkt et +.ky punkt ky +.cool punkt cool +.lr punkt lr +.bb punkt bb +.support punkt support +.taipei punkt taipei +.sl punkt sl +.company punkt company +.dj punkt dj +.social punkt social +.gm punkt gm +.xn--mgberp4a5d4ar punkt xn--mgberp4a5d4ar +.love punkt love +.cymru punkt cymru +.directory punkt directory +.games punkt games +.bio punkt bio +.wang punkt wang +.watch punkt watch +.cafe punkt cafe +.care punkt care +.supply punkt supply +.ooo punkt ooo +.group punkt group +.tools punkt tools +.sx punkt sx +.rest punkt rest +.law punkt law +.wales punkt wales +.movie punkt movie +.monster punkt monster +.studio punkt studio +.wtf punkt wtf +.direct punkt direct +.education punkt education +.ye punkt ye +.fit punkt fit +.events punkt events +.berlin punkt berlin +.services punkt services +.church punkt church +.pn punkt pn +.tc punkt tc +.tf punkt tf +.nyc punkt nyc +.town punkt town +.xn--90ais punkt xn--90ais +.xn--j1amh punkt xn--j1amh +.ist punkt ist +.bet punkt bet +.sale punkt sale +.sport punkt sport +.brussels punkt brussels +.moscow punkt moscow +.menu punkt menu +.express punkt express +.vlaanderen punkt vlaanderen +.blue punkt blue +.cars punkt cars +.deals punkt deals +.host punkt host +.academy punkt academy +.training punkt training +.vi punkt vi +.men punkt men +.team punkt team +.canon punkt canon +.leclerc punkt leclerc +.solutions punkt solutions +.community punkt community +.land punkt land +.basketball punkt basketball +.gallery punkt gallery +.sexy punkt sexy +.codes punkt codes +.swiss punkt swiss +.camp punkt camp +.software punkt software +.auction punkt auction +.nrw punkt nrw +.style punkt style +.coffee punkt coffee +.paris punkt paris +.graphics punkt graphics +.holiday punkt holiday +.wine punkt wine +.dog punkt dog +.dating punkt dating +.radio punkt radio +.pet punkt pet +.parts punkt parts +.international punkt international +.xin punkt xin +.bike punkt bike +.school punkt school +.stream punkt stream +.frl punkt frl +.tours punkt tours +.audio punkt audio +.legal punkt legal +.chat punkt chat +.xn--wgbl6a punkt xn--wgbl6a +.photo punkt photo +.surf punkt surf +.pink punkt pink +.xn--p1acf punkt xn--p1acf +.kiwi punkt kiwi +.doctor punkt doctor +.museum punkt museum +.science punkt science +.football punkt football +.xn--80asehdb punkt xn--80asehdb +.fail punkt fail +.moda punkt moda +.party punkt party +.rip punkt rip +.fan punkt fan +.loan punkt loan +.works punkt works +.business punkt business +.bayern punkt bayern +.money punkt money +.xn--9dbq2a punkt xn--9dbq2a +.show punkt show +.house punkt house +.futbol punkt futbol +.krd punkt krd +.fyi punkt fyi +.gold punkt gold +.nagoya punkt nagoya +.quebec punkt quebec +.xn--3e0b707e punkt xn--3e0b707e +.reviews punkt reviews +.farm punkt farm +.photography punkt photography +.photos punkt photos +.xn--tckwe punkt xn--tckwe +.cab punkt cab +.army punkt army +.hamburg punkt hamburg +.amsterdam punkt amsterdam +.pictures punkt pictures +.cash punkt cash +.adult punkt adult +.ski punkt ski +.poker punkt poker +.black punkt black +.fashion punkt fashion +.clinic punkt clinic +.gdn punkt gdn +.tatar punkt tatar +.exchange punkt exchange +.date punkt date +.trade punkt trade +.boutique punkt boutique +.shoes punkt shoes +.ren punkt ren +.green punkt green +.band punkt band +.ltda punkt ltda +.istanbul punkt istanbul +.health punkt health +.webcam punkt webcam +.earth punkt earth +.garden punkt garden +.yoga punkt yoga +.wien punkt wien +.ngo punkt ngo +.xn--80adxhks punkt xn--80adxhks +.xn--80aswg punkt xn--80aswg +.xn--fiqs8s punkt xn--fiqs8s +.repair punkt repair +.xn--90a3ac punkt xn--90a3ac +.fitness punkt fitness +.xn--c1avg punkt xn--c1avg +.immo punkt immo +.observer punkt observer +.vin punkt vin +.singles punkt singles +.haus punkt haus +.okinawa punkt okinawa +.corsica punkt corsica +.irish punkt irish +.university punkt university +.casino punkt casino +.how punkt how +.td punkt td +.dance punkt dance +.technology punkt technology +.barcelona punkt barcelona +.nf punkt nf +.golf punkt golf +.wf punkt wf +.estate punkt estate +.yokohama punkt yokohama +.systems punkt systems +.rugby punkt rugby +.wedding punkt wedding +.healthcare punkt healthcare +.koeln punkt koeln +.enterprises punkt enterprises +.film punkt film +.xn--node punkt xn--node +.accountant punkt accountant +.taxi punkt taxi +.tienda punkt tienda +.nr punkt nr +.toys punkt toys +.gent punkt gent +.vision punkt vision +.rent punkt rent +.fans punkt fans +.energy punkt energy +.jetzt punkt jetzt +.cards punkt cards +.domains punkt domains +.marketing punkt marketing +.soy punkt soy +.foundation punkt foundation +.horse punkt horse +.fish punkt fish +.promo punkt promo +.institute punkt institute +.restaurant punkt restaurant +.exposed punkt exposed +.flowers punkt flowers +.gift punkt gift +.jewelry punkt jewelry +.ki punkt ki +.coach punkt coach +.clothing punkt clothing +.fund punkt fund +.finance punkt finance +.dental punkt dental +.forsale punkt forsale +.tirol punkt tirol +.xn--mgbayh7gpa punkt xn--mgbayh7gpa +.family punkt family +.xn--80ao21a punkt xn--80ao21a +.gn punkt gn +.xn--d1acj3b punkt xn--d1acj3b +.madrid punkt madrid +.soccer punkt soccer +.eco punkt eco +.kitchen punkt kitchen +.consulting punkt consulting +.immobilien punkt immobilien +.xn--90ae punkt xn--90ae +.beer punkt beer +.capital punkt capital +.rio punkt rio +.miami punkt miami +.alsace punkt alsace +.archi punkt archi +.ruhr punkt ruhr +.lgbt punkt lgbt +.hosting punkt hosting +.delivery punkt delivery +.limited punkt limited +.properties punkt properties +.baby punkt baby +.bible punkt bible +.schule punkt schule +.bnpparibas punkt bnpparibas +.shopping punkt shopping +.partners punkt partners +.recipes punkt recipes +.bank punkt bank +.lighting punkt lighting +.ong punkt ong +.hm punkt hm +.srl punkt srl +.discount punkt discount +.gifts punkt gifts +.pizza punkt pizza +.vegas punkt vegas +.bot punkt bot +.builders punkt builders +.llc punkt llc +.tax punkt tax +.realty punkt realty +.equipment punkt equipment +.gmbh punkt gmbh +.tattoo punkt tattoo +.homes punkt homes +.saarland punkt saarland +.property punkt property +.kyoto punkt kyoto +.xn--y9a3aq punkt xn--y9a3aq +.solar punkt solar +.weber punkt weber +.gop punkt gop +.mom punkt mom +.fishing punkt fishing +.cricket punkt cricket +.faith punkt faith +.inc punkt inc +.rentals punkt rentals +.build punkt build +.college punkt college +.sharp punkt sharp +.camera punkt camera +.xn--h2brj9c punkt xn--h2brj9c +.cern punkt cern +.sydney punkt sydney +.dhl punkt dhl +.salon punkt salon +.cologne punkt cologne +.cyou punkt cyou +.game punkt game +.vote punkt vote +.flights punkt flights +.glass punkt glass +.melbourne punkt melbourne +.hiphop punkt hiphop +.mba punkt mba +.post punkt post +.voyage punkt voyage +.ventures punkt ventures +.courses punkt courses +.kaufen punkt kaufen +.productions punkt productions +.abudhabi punkt abudhabi +.hospital punkt hospital +.shiksha punkt shiksha +.racing punkt racing +.furniture punkt furniture +.capetown punkt capetown +.aq punkt aq +.engineering punkt engineering +.study punkt study +.luxury punkt luxury +.reisen punkt reisen +.sandvik punkt sandvik +.surgery punkt surgery +.hockey punkt hockey +.audi punkt audi +.contractors punkt contractors +.florist punkt florist +.cheap punkt cheap +.actor punkt actor +.computer punkt computer +.xn--mgbaam7a8h punkt xn--mgbaam7a8h +.coupons punkt coupons +.management punkt management +.organic punkt organic +.cooking punkt cooking +.markets punkt markets +.construction punkt construction +.voto punkt voto +.realestate punkt realestate +.xn--l1acc punkt xn--l1acc +.engineer punkt engineer +.lawyer punkt lawyer +.physio punkt physio +.yandex punkt yandex +.pharmacy punkt pharmacy +.insure punkt insure +.boston punkt boston +.xn--j6w193g punkt xn--j6w193g +.guitars punkt guitars +.saxo punkt saxo +.car punkt car +.osaka punkt osaka +.xn--mk1bu44c punkt xn--mk1bu44c +.industries punkt industries +.tennis punkt tennis +.abbott punkt abbott +.sap punkt sap +.xn--ses554g punkt xn--ses554g +.gw punkt gw +.credit punkt credit +.xn--kpry57d punkt xn--kpry57d +.supplies punkt supplies +.holdings punkt holdings +.ryukyu punkt ryukyu +.diamonds punkt diamonds +.apartments punkt apartments +.associates punkt associates +.cruises punkt cruises +.fk punkt fk +.theater punkt theater +.ceo punkt ceo +.rodeo punkt rodeo +.attorney punkt attorney +.viajes punkt viajes +.google punkt google +.auto punkt auto +.kp punkt kp +.diet punkt diet +.joburg punkt joburg +.gives punkt gives +.villas punkt villas +.financial punkt financial +.sarl punkt sarl +.catering punkt catering +.vacations punkt vacations +.rehab punkt rehab +.xn--fiqz9s punkt xn--fiqz9s +.gu punkt gu +.investments punkt investments +.qpon punkt qpon +.forex punkt forex +.gripe punkt gripe +.stockholm punkt stockholm +.career punkt career +.bingo punkt bingo +.limo punkt limo +.xn--d1alf punkt xn--d1alf +.storage punkt storage +.cleaning punkt cleaning +.dentist punkt dentist +.hiv punkt hiv +.degree punkt degree +.plumbing punkt plumbing +.citic punkt citic +.crs punkt crs +.edeka punkt edeka +.lease punkt lease +.bargains punkt bargains +.durban punkt durban +.condos punkt condos +.loans punkt loans +.xn--wgbh1c punkt xn--wgbh1c +.uol punkt uol +.accountants punkt accountants +.country punkt country +.seat punkt seat +.aws punkt aws +.yachts punkt yachts +.luxe punkt luxe +.barclays punkt barclays +.bradesco punkt bradesco +.xn--55qx5d punkt xn--55qx5d +.maison punkt maison +.kred punkt kred +.xn--czr694b punkt xn--czr694b +.xn--e1a4c punkt xn--e1a4c +.xn--ngbc5azd punkt xn--ngbc5azd +.komatsu punkt komatsu +.reise punkt reise +.xn--kput3i punkt xn--kput3i +.trading punkt trading +.claims punkt claims +.xn--q9jyb4c punkt xn--q9jyb4c +.mortgage punkt mortgage +.arpa punkt arpa +.xn--o3cw4h punkt xn--o3cw4h +.ricoh punkt ricoh +.xn--6qq986b3xl punkt xn--6qq986b3xl +.sener punkt sener +.xn--54b7fta0cc punkt xn--54b7fta0cc +.versicherung punkt versicherung +.mango punkt mango +.charity punkt charity +.feedback punkt feedback +.security punkt security +.ipiranga punkt ipiranga +.xn--qxam punkt xn--qxam +.xn--zfr164b punkt xn--zfr164b +.creditunion punkt creditunion +.afl punkt afl +.christmas punkt christmas +.broker punkt broker +.vodka punkt vodka +.xn--czru2d punkt xn--czru2d +.gay punkt gay +.tickets punkt tickets +.memorial punkt memorial +.xn--t60b56a punkt xn--t60b56a +.neustar punkt neustar +.mma punkt mma +.xn--io0a7i punkt xn--io0a7i +.xn--3ds443g punkt xn--3ds443g +.navy punkt navy +.aquarelle punkt aquarelle +.juegos punkt juegos +.km punkt km +.total punkt total +.democrat punkt democrat +.sucks punkt sucks +.tires punkt tires +.fage punkt fage +.xn--mgbab2bd punkt xn--mgbab2bd +.jll punkt jll +.blackfriday punkt blackfriday +.jcb punkt jcb +.xn--vuq861b punkt xn--vuq861b +.weir punkt weir +.aco punkt aco +.boats punkt boats +.republican punkt republican +.man punkt man +.motorcycles punkt motorcycles +.hsbc punkt hsbc +.abogado punkt abogado +.reit punkt reit +.axa punkt axa +.xn--6frz82g punkt xn--6frz82g +.sky punkt sky +.dvag punkt dvag +.insurance punkt insurance +.kpn punkt kpn +.praxi punkt praxi +.globo punkt globo +.statebank punkt statebank +.xn--fiq228c5hs punkt xn--fiq228c5hs +.bmw punkt bmw +.xn--g2xx48c punkt xn--g2xx48c +.abb punkt abb +.toray punkt toray +.hisamitsu punkt hisamitsu +.philips punkt philips +.schwarz punkt schwarz +.monash punkt monash +.airforce punkt airforce +.erni punkt erni +.latrobe punkt latrobe +.xn--czrs0t punkt xn--czrs0t \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d2428d841a8f3ab70c096e5d878ed6381cdd05fb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv @@ -0,0 +1,17 @@ +gmail g mail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast +aol +signup +brettspielversand +Sinnesbereichen +sinnesbereichen +amazon \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..57a0317f86823a8f55a0e5a3e7e4614b8a367ffa --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/electronic/symbols.tsv @@ -0,0 +1,20 @@ +. punkt +: doppelpunkt +- bindestrich +_ unterstrich +! ausrufezeichen +# raute +$ dollar zeichen +% prozent zeichen +& und +' apostroph +* asterisk ++ plus +/ slash += gleichheitszeichen +? fragezeichen +^ zirkumflex +{ linke klammer auf +} rechte klammer zu +~ tilde +, komma \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/fractions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/fractions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..50d8847a1fc305cb4c82b951c6119baed7d5f512 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/fractions.tsv @@ -0,0 +1,30 @@ +halb zwei +drittel drei +viertel vier +fünftel fünf +sechstel sechs +siebtel sieben +achtel acht +neuntel neun +zehntel zehn +elftel elf +zwölftel zwölf +dreizehntel dreizehn +vierzehntel vierzehn +fünfzehntel fünfzehn +sechzehntel sechzehn +siebzehntel siebzehn +achtzehntel achtzehn +neunzehntel neunzehn +zwanzigstel zwanzig +dreißigstel dreißig +vierzigstel vierzig +fünfzigstel fünfzig +sechzigstel sechzig +siebzigstel siebzig +achtzigstel achtzig +neunzigstel neunzig +hundertstel hundert +tausendstel tausend +millionstel million +milliardstel milliarde \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/measurements.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8292eca7fa2c0adff9c3154114b2d67a023eab14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/measurements.tsv @@ -0,0 +1,82 @@ +% prozent +f fahrenheit +c celsius +°C grad celsius +°F grad fahrenheit +K kelvin +km kilometer +m meter +cm zentimeter +mm millimeter +μm mikrometer +nm nanometer +dm dezimeter +pm pikometer +hm hektometer +ha hektar +mi meile +m² quadrat meter -0.0001 +km² quadrat kilometer -0.0001 +mm² quadrat millimeter -0.0001 +cm² quadrat zentimeter -0.0001 +m³ kubik meter -0.0001 +km³ kubik kilometer -0.0001 +mm³ kubik millimeter -0.0001 +cm³ kubik zentimeter -0.0001 +m2 quadrat meter +km2 quadrat kilometer +mm2 quadrat millimeter +cm2 quadrat zentimeter +m3 kubik meter +km3 kubik kilometer +mm3 kubik millimeter +cm3 kubik zentimeter +ft fuß +g gramm +µg mikrogramm +mg milligramm +kg kilogramm +lb pfund +oz unze +cwt zentner +gr korn +dr drachne +μg mikrogramm +pg petagramm +h stunde +s sekunde +min minute +ds decisekunde +ms millisekunde +μs mikrosekunde +hz hertz +kw kilowatt +kwh kilowattstunde +ghz gigahertz +khz kilohertz +mhz megahertz +v volt +mc megacoulomb +mA milliampere +A ampere +tw terawatt +mv millivolt +mw megawatt +gw gigawatt +ω ohm +db dezibel +gb gigabyte +kb kilobit +pb petabit +mb megabyte +kb kilobyte +tb terabyte +kv kilovolt +mv megavolt +kn kilonewton +ml milliliter +l liter +ma megaampere +bar bar +kcal kilokalorie +cal kalorie \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/suppletive.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/suppletive.tsv new file mode 100644 index 0000000000000000000000000000000000000000..449d545f67b5ed0b79b7bbb98d5ace138d0d2850 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/measure/suppletive.tsv @@ -0,0 +1,10 @@ +meile meilen +unze unzen +drachne drachnen +stunde stunden +sekunde sekunden +minute minuten +minute minuten +bit bits +byte bytes +kalorie kalorien \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fa6c618ce5a837bbcda12fb9338dfbc350c6942b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency.tsv @@ -0,0 +1,25 @@ +€ euro +$ dollar +$ us dollar +£ pfund +₩ won +nzd neuseeland dollar +rs rupie +chf schweizer franken +dkk dänische krone +fim finnische mark +aed dirham +¥ yen +czk tschechische krone +mro ouguiya +pkr pakistanische rupie +crc colon +hkd hong kong dollar +npr nepalesische rupee +awg aruba florin +nok norwegische krone +tzs tansania schilling +sek schwedisch krone +cyp zypern pfund +dm d-mark +dm deutsche mark \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_plural.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_plural.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9818ee7437939e98aa1e9dd98cbea43e1991ca8d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_plural.tsv @@ -0,0 +1,3 @@ +$ cent +€ cent +£ pence \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_singular.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_singular.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fe629d87b693de1180ee75c642190687a754e126 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/money/currency_minor_singular.tsv @@ -0,0 +1,3 @@ +$ cent +€ cent +£ penny \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/abbr_to_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/abbr_to_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..81a323845b149ef9b7a29ef3cc0d0fbe91582e25 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/abbr_to_name.tsv @@ -0,0 +1,13 @@ +jan januar +feb februar +mär märz +apr april +mai mai +jun juni +jul juli +aug august +sep september +sept september +okt oktober +nov november +dez dezember \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/numbers.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/numbers.tsv new file mode 100644 index 0000000000000000000000000000000000000000..012afcc2a34d3747f0d5f9ec73f5635f4ad231fb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/months/numbers.tsv @@ -0,0 +1,24 @@ +1 januar +2 februar +3 märz +4 april +5 mai +6 juni +7 juli +8 august +9 september +10 oktober +11 november +12 dezember +01 januar +02 februar +03 märz +04 april +05 mai +06 juni +07 juli +08 august +09 september +10 oktober +11 november +12 dezember \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7634e5ac545e4d33af493cf16c4652c8c964bc49 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/digit.tsv @@ -0,0 +1,8 @@ +zwei 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ones.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ones.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7d12cd59b1d874b5fb550f3ce56f000c6530c76d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ones.tsv @@ -0,0 +1,3 @@ +eine 1 +ein 1 +eins 1 -0.0001 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/quantities.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..712fe0f3851440d0f42b5f4aa9e5b461f719290e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/quantities.tsv @@ -0,0 +1,12 @@ +million +millionen +milliarde +milliarden +billion +billionen +billiarde +billiarden +trillion +trillionen +trilliarde +trilliarde \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..854712ea61566a3374a35f45e360012b49be9161 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +zehn 10 +elf 11 +zwölf 12 +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..28a3f80324558f078eeb46fe7b61ec3c578ed687 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e27b1c6405a456a712addc57b4199493bd5133a7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/numbers/zero.tsv @@ -0,0 +1 @@ +null 0 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..31c91f635d8066c31f164456b776f139c5012d96 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/digit.tsv @@ -0,0 +1,9 @@ +ers eins +zwei zwei +drit drei +vier vier +fünf fünf +sechs sechs +sieb sieben +ach acht +neun neun \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/thousands.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/thousands.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a97b94c744da416806591ac180ebb07e2134bae8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/thousands.tsv @@ -0,0 +1,4 @@ +hunderts hundert +tausends tausend +millions million +milliards milliarde \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4ab2bd5080460ee49a1858839f4b32fdb8098e6f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/ordinals/ties.tsv @@ -0,0 +1,8 @@ +zwanzigs zwanzig +dreißigs dreißig +vierzigs vierzig +fünfzigs fünfzig +sechzigs sechzig +siebzigs siebzig +achtzigs achtzig +neunzigs neunzig \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e681afc6c25c71eef187fc7a300df91cb411149b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to.tsv @@ -0,0 +1,12 @@ +1 12 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to_night.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to_night.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b7765ac2a3c6158197b46e5886345e0e08b6b3ac --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/hour_to_night.tsv @@ -0,0 +1,13 @@ +12 0 +1 13 +2 14 +3 15 +4 16 +5 17 +6 18 +7 19 +8 20 +9 21 +10 22 +11 23 +12 24 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/minute_to.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/minute_to.tsv new file mode 100644 index 0000000000000000000000000000000000000000..edab4d5b03d6fd6c8187ea76523f61f0df3410b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/minute_to.tsv @@ -0,0 +1,59 @@ +1 59 +2 58 +3 57 +4 56 +5 55 +6 54 +7 53 +8 52 +9 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 9 +52 8 +53 7 +54 6 +55 5 +56 4 +57 3 +58 2 +59 1 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/time_zone.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/time_zone.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3e0ade467a844e4f040d51c3334341b729432174 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/time/time_zone.tsv @@ -0,0 +1,7 @@ +cst c s t +cet c e t +pst p s t +est e s t +pt p t +et e t +gmt g m t diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9ac21da340603f3a2d4bca676039b7bd72f8d26a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/data/whitelist.tsv @@ -0,0 +1,21 @@ +z.B. zum beispiel +d.h. dass heißt +Dr. doktor +Mr. mister +Mrs. misses +Ms. miss +Nr. nummer +2D zwei-D +2-D zwei-D +3D drei-D +3-D drei-D +3-D-Mammogram drei-D-Mammogram +3D-Mammogram drei-D-Mammogram +2-D-Mammogram zwei-D-Mammogram +2D-Mammogram zwei-D-Mammogram +3-D-Mammographie drei-D-Mammographie +3D-Mammographie drei-D-Mammographie +2-D-Mammographie zwei-D-Mammographie +2D-Mammographie zwei-D-Mammographie +3-D-Drucker drei-D-Drucker +3D-Drucker drei-D-Drucker \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..8f9e7afbcf8cbe1ec1173db482dc61249c9aefec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/cardinal.py @@ -0,0 +1,200 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + +AND = "und" + + +def get_ties_digit(digit_path: str, tie_path: str) -> 'pynini.FstLike': + """ + getting all inverse normalizations for numbers between 21 - 100 + + Args: + digit_path: file to digit tsv + tie_path: file to tie tsv, e.g. 20, 30, etc. + Returns: + res: fst that converts numbers to their verbalization + """ + + digits = defaultdict(list) + ties = defaultdict(list) + for k, v in load_labels(digit_path): + digits[v].append(k) + digits["1"] = ["ein"] + + for k, v in load_labels(tie_path): + ties[v].append(k) + + d = [] + for i in range(21, 100): + s = str(i) + if s[1] == "0": + continue + + for di in digits[s[1]]: + for ti in ties[s[0]]: + word = di + f" {AND} " + ti + d.append((word, s)) + + res = pynini.string_map(d) + return res + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "101" -> cardinal { integer: "ein hundert und zehn" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")).invert() + graph_digit = graph_digit_no_one | graph_one + self.digit = (graph_digit | graph_zero).optimize() + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).invert() + + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert() + # separator = "." + + def tens_no_zero(): + return ( + pynutil.delete("0") + graph_digit + | get_ties_digit( + get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ties.tsv") + ).invert() + | graph_teen + | (graph_ties + pynutil.delete("0")) + ) + + def hundred_non_zero(): + return (graph_digit_no_one + insert_space | pynini.cross("1", "ein ")) + pynutil.insert("hundert") + ( + pynini.closure(insert_space + pynutil.insert(AND, weight=0.0001), 0, 1) + insert_space + tens_no_zero() + | pynutil.delete("00") + ) | pynutil.delete("0") + tens_no_zero() + + def thousand(): + return (hundred_non_zero() + insert_space + pynutil.insert("tausend") | pynutil.delete("000")) + ( + insert_space + hundred_non_zero() | pynutil.delete("000") + ) + + optional_plural_quantity_en = pynini.closure(pynutil.insert("en", weight=-0.0001), 0, 1) + optional_plural_quantity_n = pynini.closure(pynutil.insert("n", weight=-0.0001), 0, 1) + graph_million = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("million") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_billion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("milliarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + + graph_trillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("billion") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_quadrillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("billiarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + + graph_quintillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("trillion") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_sextillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("trilliarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + graph = pynini.union( + graph_sextillion + + insert_space + + graph_quintillion + + insert_space + + graph_quadrillion + + insert_space + + graph_trillion + + insert_space + + graph_billion + + insert_space + + graph_million + + insert_space + + thousand() + ) + + fix_syntax = [ + ("eins tausend", "ein tausend"), + ("eins millionen", "eine million"), + ("eins milliarden", "eine milliarde"), + ("eins billionen", "eine billion"), + ("eins billiarden", "eine billiarde"), + ] + fix_syntax = pynini.union(*[pynini.cross(*x) for x in fix_syntax]) + self.graph = ( + ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(fix_syntax, "[BOS]", "", NEMO_SIGMA) + ) + self.graph |= graph_zero | pynini.cross("1", "eins") + + # self.graph = pynini.cdrewrite(pynutil.delete(separator), "", "", NEMO_SIGMA) @ self.graph + self.graph = self.graph.optimize() + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**3 + @ hundred_non_zero() + ) | pynini.cross("1", "eins") + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + self.graph_hundred_component_at_least_one_none_zero_digit.optimize() + ) + + self.two_digit_non_zero = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ self.graph_hundred_component_at_least_one_none_zero_digit + ) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..35e96409454117099930799255273d78f3ece89c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/date.py @@ -0,0 +1,142 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + TO_LOWER, + GraphFst, + insert_space, +) + +graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))).optimize() +graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() +ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))).optimize() +delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + +def get_year_graph(cardinal: GraphFst) -> 'pynini.FstLike': + """ + Returns year verbalizations as fst + + < 2000 neunzehn (hundert) (vier und zwanzig), >= 2000 regular cardinal + **00 ** hundert + + Args: + delete_leading_zero: removed leading zero + cardinal: cardinal GraphFst + """ + + year_gt_2000 = (pynini.union("21", "20") + NEMO_DIGIT**2) @ cardinal.graph + + graph_two_digit = delete_leading_zero @ cardinal.two_digit_non_zero + hundred = pynutil.insert("hundert") + graph_double_double = ( + (pynini.accep("1") + NEMO_DIGIT) @ graph_two_digit + + insert_space + + pynini.closure(hundred + insert_space, 0, 1) + + graph_two_digit + ) + # for 20** + graph_double_double |= pynini.accep("20") @ graph_two_digit + insert_space + graph_two_digit + graph = ( + graph_double_double + | (pynini.accep("1") + NEMO_DIGIT) @ graph_two_digit + insert_space + pynutil.delete("00") + hundred + | year_gt_2000 + ) + return graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "01.04.2010" -> date { day: "erster" month: "april" year: "zwei tausend zehn" preserve_order: true } + "1994" -> date { year: "neunzehn vier und neuzig" } + "1900" -> date { year: "neunzehn hundert" } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + month_abbr_graph = load_labels(get_abs_path("data/months/abbr_to_name.tsv")) + number_to_month = pynini.string_file(get_abs_path("data/months/numbers.tsv")).optimize() + month_graph = pynini.union(*[x[1] for x in month_abbr_graph]).optimize() + month_abbr_graph = pynini.string_map(month_abbr_graph) + month_abbr_graph = ( + pynutil.add_weight(month_abbr_graph, weight=0.0001) + | ((TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_abbr_graph) + ) + pynini.closure(pynutil.delete(".", weight=-0.0001), 0, 1) + + self.month_abbr = month_abbr_graph + month_graph |= (TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_graph + # jan.-> januar, Jan-> januar, januar-> januar + month_graph |= month_abbr_graph + + numbers = cardinal.graph_hundred_component_at_least_one_none_zero_digit + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + # 01, 31, 1 + digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers + day = (pynutil.insert("day: \"") + digit_day + pynutil.insert("\"")).optimize() + + digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = digit_month @ number_to_month + digit_month @= numbers + + month_name = (pynutil.insert("month: \"") + month_graph + pynutil.insert("\"")).optimize() + month_number = ( + pynutil.insert("month: \"") + + (pynutil.add_weight(digit_month, weight=0.0001) | number_to_month) + + pynutil.insert("\"") + ).optimize() + + # prefer cardinal over year + year = pynutil.add_weight(get_year_graph(cardinal=cardinal), weight=0.001) + self.year = year + + year_only = pynutil.insert("year: \"") + year + pynutil.insert("\"") + + graph_dmy = ( + day + + pynutil.delete(".") + + pynini.closure(pynutil.delete(" "), 0, 1) + + insert_space + + month_name + + pynini.closure(pynini.accep(" ") + year_only, 0, 1) + ) + + separators = ["."] + for sep in separators: + year_optional = pynini.closure(pynini.cross(sep, " ") + year_only, 0, 1) + new_graph = day + pynini.cross(sep, " ") + month_number + year_optional + graph_dmy |= new_graph + + dash = "-" + day_optional = pynini.closure(pynini.cross(dash, " ") + day, 0, 1) + graph_ymd = year_only + pynini.cross(dash, " ") + month_number + day_optional + + final_graph = graph_dmy + pynutil.insert(" preserve_order: true") + final_graph |= year_only + final_graph |= graph_ymd + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..138f6d03007ab70294f95bbcf918ead26f8351fa --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/decimal.py @@ -0,0 +1,83 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 1 million -> integer_part: "eine" quantity: "million" + e.g. 1.4 million -> integer_part: "eins" fractional_part: "vier" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + numbers = cardinal_up_to_hundred + + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + pynini.accep(" ") + + pynutil.insert("quantity: \"") + + quantities + + pynutil.insert("\"") + ) + res |= decimal + pynini.accep(" ") + pynutil.insert("quantity: \"") + quantities + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -11,4006 billion -> decimal { negative: "true" integer_part: "elf" fractional_part: "vier null null sechs" quantity: "billion" preserve_order: true } + 1 billion -> decimal { integer_part: "eins" quantity: "billion" preserve_order: true } + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_digit |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digit |= pynini.cross("1", "eins") + self.graph = graph_digit + pynini.closure(insert_space + graph_digit).optimize() + + point = pynutil.delete(",") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal.graph + pynutil.insert("\"") + final_graph_wo_sign = self.graph_integer + point + insert_space + self.graph_fractional + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + final_graph = optional_graph_negative + self.final_graph_wo_negative + final_graph += pynutil.insert(" preserve_order: true") + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..e04a441e1cebde075d1a504ac9edc0590515848b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/electronic.py @@ -0,0 +1,83 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + # all symbols + symbols_no_period = pynini.difference(symbols, dot) # alphabet of accepted symbols excluding the '.' + accepted_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1 + ) # alphabet of accepted chars excluding the '.' + all_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols), 1 + ) # alphabet of accepted chars including the '.' + + # domains + domain = dot + accepted_characters + domain_graph = ( + pynutil.insert('domain: "') + + (accepted_characters + pynini.closure(domain, 1)) + + dot.ques + + pynutil.insert('"') + ) + + # email + username = pynutil.insert('username: "') + all_characters + pynutil.insert('"') + pynini.cross("@", " ") + email = username + domain_graph + + # social media tags + tag = ( + pynini.cross("@", "") + + pynutil.insert('username: "') + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + dot.ques + + pynutil.insert('"') + ) + + # url + protocol_start = pynini.accep("https://") | pynini.accep("http://") + protocol_end = pynini.accep("www.") + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') + url = protocol + insert_space + (domain_graph) + + graph = url | domain_graph | email | tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..ceea4ab9f3b9ab0f2ce5765a0d85f51645317573 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/fraction.py @@ -0,0 +1,55 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/6" -> + fraction { integer: "drei und zwanzig" numerator: "vier" denominator: "sechs" preserve_order: true } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + self.optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1 + ) + self.integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + self.numerator = ( + pynutil.insert("numerator: \"") + cardinal_graph + pynini.cross(pynini.union("/", " / "), "\" ") + ) + self.denominator = pynutil.insert("denominator: \"") + cardinal_graph + pynutil.insert("\"") + + self.graph = ( + self.optional_graph_negative + + pynini.closure(self.integer + pynini.accep(" "), 0, 1) + + self.numerator + + self.denominator + ) + + graph = self.graph + pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..ff1be6cf108f66e5445c0e0b63a7568d8b3a7ca1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/measure.py @@ -0,0 +1,190 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + insert_space, +) + +unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) +suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv")) + + +def singular_to_plural(): + # plural endung n/en maskuline Nomen mit den Endungen e, ent, and, ant, ist, or + _n = NEMO_SIGMA + pynini.union("e") + pynutil.insert("n") + _en = ( + NEMO_SIGMA + + pynini.union("ent", "and", "ant", "ist", "or", "ion", "ik", "heit", "keit", "schaft", "tät", "ung") + + pynutil.insert("en") + ) + _nen = NEMO_SIGMA + pynini.union("in") + (pynutil.insert("e") | pynutil.insert("nen")) + _fremd = NEMO_SIGMA + pynini.union("ma", "um", "us") + pynutil.insert("en") + # maskuline Nomen mit den Endungen eur, ich, ier, ig, ling, ör + _e = NEMO_SIGMA + pynini.union("eur", "ich", "ier", "ig", "ling", "ör") + pynutil.insert("e") + _s = NEMO_SIGMA + pynini.union("a", "i", "o", "u", "y") + pynutil.insert("s") + + graph_plural = plurals._priority_union( + suppletive, pynini.union(_n, _en, _nen, _fremd, _e, _s), NEMO_SIGMA + ).optimize() + + return graph_plural + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "2,4 oz" -> measure { cardinal { integer_part: "zwei" fractional_part: "vier" units: "unzen" preserve_order: true } } + "1 oz" -> measure { cardinal { integer: "zwei" units: "unze" preserve_order: true } } + "1 million oz" -> measure { cardinal { integer: "eins" quantity: "million" units: "unze" preserve_order: true } } + This class also converts words containing numbers and letters + e.g. "a-8" —> "a acht" + e.g. "1,2-a" —> "ein komma zwei a" + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = graph_unit_singular @ pynini.cdrewrite(convert_space(suppletive), "", "[EOS]", NEMO_SIGMA) + optional_graph_negative = pynini.closure("-", 0, 1) + + graph_unit_denominator = ( + pynini.cross("/", "pro") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + + (graph_unit_plural + (optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + unit_singular_graph = ( + pynutil.insert("units: \"") + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + + subgraph_cardinal = ( + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_plural + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) + @ cardinal.fst + @ pynini.cdrewrite(pynini.cross("eins", "ein"), "", "", NEMO_SIGMA) + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_singular_graph + ) + + subgraph_fraction = fraction.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + + cardinal_dash_alpha = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" }") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.delete('-') + + pynutil.insert(" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + cardinal_times = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | subgraph_fraction + | cardinal_times + ) + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..18c0e6598174f591784387fb1cecd1cca99d325f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/money.py @@ -0,0 +1,149 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + insert_space, +) + +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) +min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "€1" -> money { currency_maj: "euro" integer_part: "ein"} + "€1,000" -> money { currency_maj: "euro" integer_part: "ein" } + "€1,001" -> money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"} + "£1,4" -> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true} + -> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true} + "£0,01" -> money { fractional_part: "ein" currency_min: "penny" preserve_order: true} + "£0,01 million" -> money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"} + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.fst + + maj_singular_labels = load_labels(get_abs_path("data/money/currency.tsv")) + maj_singular_graph = convert_space(maj_singular) + maj_plural_graph = maj_singular_graph + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_singular_graph + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_plural_graph + pynutil.insert("\"") + + optional_delete_fractional_zeros = pynini.closure( + pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), 0, 1 + ) + graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "ein") + pynutil.insert("\"") + + # only for decimals where third decimal after comma is non-zero or with quantity + decimal_delete_last_zeros = ( + pynini.closure(NEMO_DIGIT, 1) + + pynini.accep(",") + + pynini.closure(NEMO_DIGIT, 2) + + (NEMO_DIGIT - "0") + + pynini.closure(pynutil.delete("0")) + ) + decimal_with_quantity = NEMO_SIGMA + NEMO_ALPHA + graph_decimal = ( + graph_maj_plural + insert_space + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") + ) + + graph_integer_only = graph_maj_singular + insert_space + graph_integer_one + graph_integer_only |= graph_maj_plural + insert_space + graph_integer + + graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert(" currency_min: \"") + min_singular + pynutil.insert("\"") + graph_min_plural = pynutil.insert(" currency_min: \"") + min_plural + pynutil.insert("\"") + + # format ** euro ** cent + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + integer_plus_maj = graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural + integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular + # non zero integer part + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "ein") + graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"") + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_non_zero + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural + fractional_plus_min |= ( + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular + ) + + decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(",", " ") + fractional_plus_min + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(",", " ") + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.two_digit_non_zero + + pynutil.insert("\""), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = ( + pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | decimal_graph_with_minor + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..389430dfae002501585e84cb17ebe8c57ea87e5f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/ordinal.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted from https://github.com/google/TextNormalizationCoveringGrammars +# Russian minimally supervised number grammar. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "2." -> ordinal { integer: "zwei" } } + "2tes" -> ordinal { integer: "zwei" } } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic=False): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + endings = ["ter", "tes", "tem", "te", "ten"] + self.graph = ( + ( + pynini.closure(NEMO_DIGIT | pynini.accep(".")) + + pynutil.delete(pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")) + ) + @ cardinal_graph + ).optimize() + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..2793869748813473704b1db5bc68b3761c4cb0ed --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/telephone.py @@ -0,0 +1,73 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone, which includes country code, number part and extension + + E.g + "+49 1234-1233" -> telephone { country_code: "plus neun und vierzig" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true } + "(012) 1234-1233" -> telephone { country_code: "null eins zwei" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true } + (0**) + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + graph_digit_no_zero = pynini.invert( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + ).optimize() | pynini.cross("1", "eins") + graph_digit = graph_digit_no_zero | graph_zero + + numbers_with_single_digits = pynini.closure(graph_digit + insert_space) + graph_digit + + two_digit_and_zero = (NEMO_DIGIT**2 @ cardinal.two_digit_non_zero) | graph_zero + # def add_space_after_two_digit(): + # return pynini.closure(two_digit_and_zero + insert_space) + ( + # two_digit_and_zero + # ) + + country_code = pynini.closure(pynini.cross("+", "plus "), 0, 1) + two_digit_and_zero + country_code |= ( + pynutil.delete("(") + graph_zero + insert_space + numbers_with_single_digits + pynutil.delete(")") + ) + country_code |= graph_zero + insert_space + numbers_with_single_digits + + country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"") + + del_separator = pynini.cross(pynini.union("-", " "), " ") + # numbers_with_two_digits = pynini.closure(graph_digit + insert_space) + add_space_after_two_digit() + pynini.closure(insert_space + graph_digit) + # numbers = numbers_with_two_digits + pynini.closure(del_separator + numbers_with_two_digits, 0, 1) + numbers = numbers_with_single_digits + pynini.closure(del_separator + numbers_with_single_digits, 0, 1) + number_length = pynini.closure((NEMO_DIGIT | pynini.union("-", " ", ")", "(")), 7) + number_part = pynini.compose(number_length, numbers) + number = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + + graph = country_code + pynini.accep(" ") + number + self.graph = graph + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..a01c90902f98585baed6ec67baca5ff32332937b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/time.py @@ -0,0 +1,98 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + "02:15 Uhr est" -> time { hours: "2" minutes: "15" zone: "e s t"} + "2 Uhr" -> time { hours: "2" } + "09:00 Uhr" -> time { hours: "2" } + "02:15:10 Uhr" -> time { hours: "2" minutes: "15" seconds: "10"} + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + final_suffix = pynutil.delete(" ") + pynutil.delete("Uhr") | pynutil.delete("uhr") + time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) + + labels_hour = [str(x) for x in range(0, 25)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (pynutil.delete("0").ques | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + graph_hour = pynini.union(*labels_hour) + + graph_minute_single = pynini.union(*labels_minute_single) + graph_minute_double = pynini.union(*labels_minute_double) + + final_graph_hour_only = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"') + final_graph_hour = ( + pynutil.insert('hours: "') + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert('"') + ) + final_graph_minute = ( + pynutil.insert('minutes: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + final_graph_second = ( + pynutil.insert('seconds: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + final_time_zone_optional = pynini.closure( + pynini.accep(" ") + pynutil.insert('zone: "') + convert_space(time_zone_graph) + pynutil.insert('"'), + 0, + 1, + ) + + # Accepts the following formats: 02:30 Uhr, 02.30 Uhr, 2:30 Uhr, 2.30 Uhr + graph_hm = ( + final_graph_hour + + (pynutil.delete(":") | pynutil.delete(".")) + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + final_suffix + + final_time_zone_optional + ) + + # 10:30:05 Uhr, + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", ' minutes: "0"') | (insert_space + final_graph_minute)) + + pynutil.delete(":") + + (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second)) + + final_suffix + + final_time_zone_optional + + pynutil.insert(" preserve_order: true") + ) + + # 2 Uhr est + graph_h = final_graph_hour_only + final_suffix + final_time_zone_optional + final_graph = (graph_hm | graph_h | graph_hms).optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..20eb2853e5d58a1ff26b9f382727528133a4cce7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py @@ -0,0 +1,151 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_de_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic) + ordinal_graph = self.ordinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + self.fraction = FractionFst(cardinal=self.cardinal, deterministic=deterministic) + fraction_graph = self.fraction.fst + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, + ) + measure_graph = self.measure.fst + self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + word_graph = WordFst(deterministic=deterministic).fst + self.time = TimeFst(deterministic=deterministic) + time_graph = self.time.fst + self.telephone = TelephoneFst(cardinal=self.cardinal, deterministic=deterministic) + telephone_graph = self.telephone.fst + self.electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = self.electronic.fst + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + money_graph = self.money.fst + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.11) + ) + + classify |= pynutil.add_weight(word_graph, 100) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..fc8f0f36a5bd277b4d2770caa2a899cae0e171c7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/whitelist.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "Mr." -> tokens { name: "mister" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..e3d77b4dfe6283db85bf9b1769e01ffc44e1306c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. sleep -> tokens { name: "sleep" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..68b9a65d2f9cddf61c15791afdef34989c27f057 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/utils.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logger.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..a9010adff8594b268f4a35b888948e015dc89467 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "zwei" } -> "zwei" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "minus "), 0, 1) + self.optional_sign = optional_sign + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + self.numbers = integer + graph = optional_sign + self.numbers + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..f28e803d9878a1dc738fbf9c0c42d962be603c1d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/date.py @@ -0,0 +1,62 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "vier" month: "april" year: "zwei tausend zwei" } -> "vierter april zwei tausend zwei" + date { day: "vier" month: "mai" year: "zwei tausend zwei" } -> "vierter mai zwei tausend zwei" + + Args: + ordinal: ordinal verbalizer GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day_cardinal = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + day = day_cardinal @ pynini.cdrewrite(ordinal.ordinal_stem, "", "[EOS]", NEMO_SIGMA) + pynutil.insert("ter") + + months_names = pynini.union(*[x[1] for x in load_labels(get_abs_path("data/months/abbr_to_name.tsv"))]) + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + final_month = month @ months_names + final_month |= month @ pynini.difference(NEMO_SIGMA, months_names) @ pynini.cdrewrite( + ordinal.ordinal_stem, "", "[EOS]", NEMO_SIGMA + ) + pynutil.insert("ter") + + year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # day month year + graph_dmy = day + pynini.accep(" ") + final_month + pynini.closure(pynini.accep(" ") + year, 0, 1) + graph_dmy |= final_month + pynini.accep(" ") + year + + self.graph = graph_dmy | year + final_graph = self.graph + delete_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..eed6f45194d1b218af93e57e3f10bcfa53b683db --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.decimal import quantities +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "elf" fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen + decimal { integer_part: "eins" quantity: "billion" } -> eins billion + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + delete_space = pynutil.delete(" ") + self.optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "minus ") + delete_space, 0, 1) + self.integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + self.fractional = pynutil.insert(" komma ") + self.fractional_default + + self.quantity = ( + delete_space + insert_space + pynutil.delete("quantity: \"") + quantities + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + graph = self.optional_sign + ( + self.integer + self.quantity | self.integer + delete_space + self.fractional + self.optional_quantity + ) + + self.numbers = graph + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..3b6848a3b42b190f2f3b57863ca1a399b6fafeab --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/electronic.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, + insert_space, +) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c at hotmail punkt com" + -> "a b c at h o t m a i l punkt c o m" + -> "a b c at hotmail punkt c o m" + -> "a b c at h o t m a i l punkt com" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit_no_zero = pynini.invert( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + ).optimize() | pynini.cross("1", "eins") + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + graph_digit = graph_digit_no_zero | graph_zero + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize() + server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) + domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + ( + NEMO_NOT_QUOTE - pynini.accep(" ") + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"') + user_name @= verbalize_characters + + convert_defaults = (pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common) | domain_common + domain = convert_defaults + pynini.closure(insert_space + convert_defaults) + domain @= verbalize_characters + + domain = pynutil.delete('domain: "') + domain + pynutil.delete('"') + protocol = ( + pynutil.delete('protocol: "') + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete('"') + ) + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("at ") + domain | (pynutil.insert("at ") + user_name) + ) + + # normalizes sentence-final periods following URLs + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + preserve_final_period = pynini.cdrewrite(pynini.cross(" punkt", "."), "", "[EOS]", NEMO_SIGMA) + + delete_tokens = delete_tokens @ preserve_final_period + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..3d8d4a90ee6eb0f4b947f420e6b9f572918e4a67 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/fraction.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + insert_space, +) + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. fraction { integer: "drei" numerator: "eins" denominator: "zwei" }-> drei ein halb + e.g. fraction { numerator: "vier" denominator: "zwei" } -> vier halbe + e.g. fraction { numerator: "drei" denominator: "vier" } -> drei viertel + + Args: + ordinal: ordinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "minus ") + pynutil.delete(" "), 0, 1) + change_one = pynini.cdrewrite( + pynutil.add_weight(pynini.cross("eins", "ein"), weight=-0.0001), "[BOS]", "[EOS]", NEMO_SIGMA + ) + change_numerator_two = pynini.cdrewrite(pynini.cross("zweitel", "halbe"), "[BOS]", "[EOS]", NEMO_SIGMA) + integer = pynutil.delete("integer_part: \"") + change_one + pynutil.delete("\" ") + numerator = pynutil.delete("numerator: \"") + change_one + pynutil.delete("\" ") + denominator = ( + pynutil.delete("denominator: \"") + + pynini.closure(NEMO_NOT_QUOTE) + @ ( + pynini.cdrewrite(pynini.closure(ordinal.ordinal_stem, 0, 1), "", "[EOS]", NEMO_SIGMA) + + pynutil.insert("tel") + ) + @ change_numerator_two + + pynutil.delete("\"") + ) + + integer += insert_space + pynini.closure(pynutil.insert("und ", weight=0.001), 0, 1) + + denominator_one_half = pynini.cdrewrite(pynini.cross("ein halbe", "ein halb"), "[BOS]", "[EOS]", NEMO_SIGMA) + + fraction_default = (numerator + insert_space + denominator) @ denominator_one_half + + self.graph = optional_sign + pynini.closure(integer, 0, 1) + fraction_default + + graph = self.graph + delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..d67433db4c84e484cc0cb8748613c3b7572cf0a0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/measure.py @@ -0,0 +1,54 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen" + measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen" + + Args: + decimal: decimal GraphFst + cardinal: cardinal GraphFst + fraction: fraction GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = decimal.fst + graph_cardinal = cardinal.fst + graph_fraction = fraction.fst + + graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit + + graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac33a8ba6ddb6cb28c04ad82705ae750ae865c0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/money.py @@ -0,0 +1,74 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "euro" integer_part: "ein"} -> "ein euro" + money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"} -> "eins komma null null eins euro" + money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true} -> "ein pfund vierzig" + money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true} -> "ein pfund vierzig pence" + money { fractional_part: "ein" currency_min: "penny" preserve_order: true} -> "ein penny" + money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"} -> "null komma null eins million pfund" + + Args: + decimal: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + keep_space = pynini.accep(" ") + + maj = pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + min = pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_add_and = pynini.closure(pynutil.insert("und "), 0, 1) + + # *** currency_maj + graph_integer = integer_part + keep_space + maj + + # *** currency_maj + (***) | ((und) *** current_min) + graph_integer_with_minor = ( + integer_part + + keep_space + + maj + + keep_space + + (fractional_part | (optional_add_and + fractional_part + keep_space + min)) + + delete_preserve_order + ) + + # *** komma *** currency_maj + graph_decimal = decimal.fst + keep_space + maj + + # *** current_min + graph_minor = fractional_part + keep_space + min + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..84f85877557b93a043ec590db9ba9c98785a4dc2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py @@ -0,0 +1,54 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing roman numerals + e.g. ordinal { integer: "vier" } } -> "vierter" + -> "viertes" ... + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")).invert() + graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")).invert() + graph_thousands = pynini.string_file(get_abs_path("data/ordinals/thousands.tsv")).invert() + + graph = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + suffixes = pynini.union("ten", "tem", "ter", "tes", "te") + convert_rest = pynutil.insert(suffixes, weight=0.01) + self.ordinal_stem = graph_digit | graph_ties | graph_thousands + + suffix = pynini.cdrewrite( + pynini.closure(self.ordinal_stem, 0, 1) + convert_rest, + "", + "[EOS]", + NEMO_SIGMA, + ).optimize() + self.graph = pynini.compose(graph, suffix) + self.suffix = suffix + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..f5af1ff5f29ca1e3ff7038574d31ec5098640387 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/telephone.py @@ -0,0 +1,43 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" } + -> "plus neun und vierzig null eins eins eins null null null" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + self.graph = country_code + pynini.accep(" ") + number_part + + graph = self.graph + delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..5b5faab0b4a433fa6378006a1b32769d1758cd6e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/time.py @@ -0,0 +1,121 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_preserve_order, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing electronic, e.g. + time { hours: "2" minutes: "15"} -> "zwei uhr fünfzehn" + time { minutes: "15" hours: "2" } -> "viertel nach zwei" + time { minutes: "15" hours: "2" } -> "fünfzehn nach zwei" + time { hours: "14" minutes: "15"} -> "vierzehn uhr fünfzehn" + time { minutes: "15" hours: "14" } -> "viertel nach zwei" + time { minutes: "15" hours: "14" } -> "fünfzehn nach drei" + time { minutes: "45" hours: "14" } -> "viertel vor drei" + + Args: + cardinal_tagger: cardinal_tagger tagger GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + # add weight so when using inverse text normalization this conversion is depriotized + night_to_early = pynutil.add_weight( + pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))).optimize(), weight=0.0001 + ) + hour_to = pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to.tsv"))).optimize() + minute_to = pynini.invert(pynini.string_file(get_abs_path("data/time/minute_to.tsv"))).optimize() + time_zone_graph = pynini.invert( + convert_space(pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/time_zone.tsv"))])) + ) + + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + number_verbalization = graph_zero | cardinal_tagger.two_digit_non_zero + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + hour_verbalized = hour @ number_verbalization @ pynini.cdrewrite( + pynini.cross("eins", "ein"), "[BOS]", "[EOS]", NEMO_SIGMA + ) + pynutil.insert(" uhr") + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + zone = pynutil.delete("zone: \"") + time_zone_graph + pynutil.delete("\"") + optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1) + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + graph_hms = ( + hour_verbalized + + pynini.accep(" ") + + minute @ number_verbalization + + pynutil.insert(" minuten") + + pynini.accep(" ") + + second @ number_verbalization + + pynutil.insert(" sekunden") + + optional_zone + ) + graph_hms @= pynini.cdrewrite( + pynini.cross("eins minuten", "eine minute") | pynini.cross("eins sekunden", "eine sekunde"), + pynini.union(" ", "[BOS]"), + "", + NEMO_SIGMA, + ) + + min_30 = [str(x) for x in range(1, 31)] + min_30 = pynini.union(*min_30) + min_29 = [str(x) for x in range(1, 30)] + min_29 = pynini.union(*min_29) + + graph_h = hour_verbalized + graph_hm = hour_verbalized + pynini.accep(" ") + minute @ number_verbalization + + graph_m_past_h = ( + minute @ min_30 @ (number_verbalization | pynini.cross("15", "viertel")) + + pynini.accep(" ") + + pynutil.insert("nach ") + # + hour @ number_verbalization + + hour @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", NEMO_SIGMA) @ number_verbalization + ) + graph_m30_h = ( + minute @ pynini.cross("30", "halb") + + pynini.accep(" ") + + hour @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", NEMO_SIGMA) @ hour_to @ number_verbalization + ) + graph_m_to_h = ( + minute @ minute_to @ min_29 @ (number_verbalization | pynini.cross("15", "viertel")) + + pynini.accep(" ") + + pynutil.insert("vor ") + + hour @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", NEMO_SIGMA) @ hour_to @ number_verbalization + ) + + self.graph = ( + graph_hms + | graph_h + | graph_hm + | pynutil.add_weight(graph_m_past_h, weight=0.0001) + | pynutil.add_weight(graph_m30_h, weight=0.0001) + | pynutil.add_weight(graph_m_to_h, weight=0.0001) + ) + optional_zone + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..94f1ca6087c24f896190c4610b785cb251e86f6b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize.py @@ -0,0 +1,76 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.cardinal import CardinalFst as CardinalTagger +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal_tagger = CardinalTagger(deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(ordinal=ordinal, deterministic=deterministic) + fraction_graph = fraction.fst + date = DateFst(ordinal=ordinal) + date_graph = date.fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + time_graph = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic).fst + + graph = ( + cardinal_graph + | measure_graph + | decimal_graph + | ordinal_graph + | date_graph + | electronic_graph + | money_graph + | fraction_graph + | whitelist_graph + | telephone_graph + | time_graph + ) + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..e5e74bb9ec437d651ffeb3d4e6d95a5bb47d664d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"de_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..84a0e30a5dd6b95da74d8fcd8f9fa1cfaab9dc78 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/clean_eval_data.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/clean_eval_data.py new file mode 100644 index 0000000000000000000000000000000000000000..01c8072772a169da18dec1cf52cec5d97235db34 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/clean_eval_data.py @@ -0,0 +1,342 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +import regex as re + +from zonos2.vendor.nemo_text_processing.text_normalization.data_loader_utils import ( + EOS_TYPE, + Instance, + load_files, + training_data_to_sentences, +) + +""" +This file is for evaluation purposes. +filter_loaded_data() cleans data (list of instances) for text normalization. Filters and cleaners can be specified for each semiotic class individually. +For example, normalized text should only include characters and whitespace characters but no punctuation. + Cardinal unnormalized instances should contain at least one integer and all other characters are removed. +""" + + +class Filter: + """ + Filter class + + Args: + class_type: semiotic class used in dataset + process_func: function to transform text + filter_func: function to filter text + + """ + + def __init__(self, class_type: str, process_func: object, filter_func: object): + self.class_type = class_type + self.process_func = process_func + self.filter_func = filter_func + + def filter(self, instance: Instance) -> bool: + """ + filter function + + Args: + filters given instance with filter function + + Returns: True if given instance fulfills criteria or does not belong to class type + """ + if instance.token_type != self.class_type: + return True + return self.filter_func(instance) + + def process(self, instance: Instance) -> Instance: + """ + process function + + Args: + processes given instance with process function + + Returns: processed instance if instance belongs to expected class type or original instance + """ + if instance.token_type != self.class_type: + return instance + return self.process_func(instance) + + +def filter_cardinal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_cardinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[^0-9]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_ordinal_1(instance: Instance) -> bool: + ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized) + return ok + + +def process_ordinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[,\s]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_decimal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_decimal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_measure_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_measure_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"m2", "m²", un_normalized) + un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized) + normalized = re.sub(r"[^a-z\s]", "", normalized) + normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_money_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_money_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"a\$", r"$", un_normalized) + un_normalized = re.sub(r"us\$", r"$", un_normalized) + un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized) + un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_time_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_time_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r": ", ":", un_normalized) + un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized) + un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_plain_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_plain_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_punct_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_punct_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_date_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_date_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_letters_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_letters_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_verbatim_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_verbatim_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_digit_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_digit_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_telephone_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_telephone_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_electronic_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_electronic_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_fraction_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_fraction_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_address_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_address_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +filters = [] +filters.append(Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)) +filters.append(Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)) +filters.append(Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)) +filters.append(Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)) +filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1)) +filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1)) + +filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1)) +filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1)) +filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1)) +filters.append(Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)) +filters.append(Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)) +filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1)) +filters.append(Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)) +filters.append(Filter(class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1)) +filters.append(Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)) +filters.append(Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)) +filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True)) + + +def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]: + """ + Filters list of instances + + Args: + data: list of instances + + Returns: filtered and transformed list of instances + """ + updates_instances = [] + for instance in data: + updated_instance = False + for fil in filters: + if fil.class_type == instance.token_type and fil.filter(instance): + instance = fil.process(instance) + updated_instance = True + if updated_instance: + if verbose: + print(instance) + updates_instances.append(instance) + return updates_instances + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", type=str, default='./en_with_types/output-00001-of-00100') + parser.add_argument("--verbose", help="print filtered instances", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + file_path = args.input + + print("Loading training data: " + file_path) + instance_list = load_files([file_path]) # List of instances + filtered_instance_list = filter_loaded_data(instance_list, args.verbose) + training_data_to_sentences(filtered_instance_list) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d200df57afefb446fde9b8fcd896f7e110f367f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/address_word.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/address_word.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2e9e71615acd7b08addd723d7941a079c7267098 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/address_word.tsv @@ -0,0 +1,14 @@ +st Street +street Street +expy Expressway +fwy Freeway +hwy Highway +dr Drive +ct Court +ave Avenue +av Avenue +cir Circle +blvd Boulevard +alley Alley +way Way +jct Junction \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/state.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/state.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4eb59eb4bdef06d9e923915950063e65a0ad7745 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/address/state.tsv @@ -0,0 +1,49 @@ +Alabama AL +Alaska AK +Arizona AZ +Arkansas AR +California CA +Colorado CO +Connecticut CT +Delaware DE +Florida FL +Georgia GA +Hawaii HI +Idaho ID +Illinois IL +Indiana IND +Iowa IA +Kansas KS +Kentucky KY +Louisiana LA +Maine ME +Maryland MD +Massachusetts MA +Michigan MI +Minnesota MN +Mississippi MS +Missouri MO +Montana MT +Nebraska NE +Nevada NV +New Hampshire NH +New Jersey NJ +New Mexico NM +New York NY +North Carolina NC +North Dakota ND +Oregon OR +Pennsylvania PA +Rhode Island RI +South Carolina SC +South Dakota SD +Tennessee TN +Tennessee TENN +Texas TX +Utah UT +Vermont VT +Virginia VA +Washington WA +West Virginia WV +Wisconsin WI +Wyoming WY \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/day.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/day.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ef8dd7af285253aae82202206f5a574f7741184d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/day.tsv @@ -0,0 +1,31 @@ +one +two +three +four +five +six +seven +eight +nine +ten +eleven +twelve +thirteen +fourteen +fifteen +sixteen +seventeen +eighteen +nineteen +twenty +twenty one +twenty two +twenty three +twenty four +twenty five +twenty six +twenty seven +twenty eight +twenty nine +thirty +thirty one \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_abbr.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e70e41045b3bc9b693a0e8791cfecfa2001cf4fc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_abbr.tsv @@ -0,0 +1,12 @@ +jan january +feb february +mar march +apr april +jun june +jul july +aug august +sep september +sept september +oct october +nov november +dec december diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1c05ac02a39e1e568ff94dfd7a08f2670573aed6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_name.tsv @@ -0,0 +1,12 @@ +january +february +march +april +may +june +july +august +september +october +november +december diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_number.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_number.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f0cacd5d1ed32680df0dd38af4354a89b37b0223 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/month_number.tsv @@ -0,0 +1,24 @@ +1 january +2 february +3 march +4 april +5 may +6 june +7 july +8 august +9 september +10 october +11 november +12 december +01 january +02 february +03 march +04 april +05 may +06 june +07 july +08 august +09 september +10 october +11 november +12 december \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/year_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/year_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8c799ddbcfba20ea267a9cd3951e563a3fdf55f0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/date/year_suffix.tsv @@ -0,0 +1,16 @@ +A. D AD +A.D AD +a. d AD +a.d AD +a. d. AD +a.d. AD +B. C BC +B.C BC +b. c BC +b.c BC +A. D. AD +A.D. AD +B. C. BC +B.C. BC +b. c. BC +b.c. BC diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c42d25f1395a43949b9a2b01c1d14ad6751a734e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv @@ -0,0 +1,5 @@ +card ending in +credit card number +credit card +debit card number +debit card \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..01da3f0bde9022f97e71724af1b995fc6b5cb96a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv @@ -0,0 +1,13 @@ +.com dot com +.org dot org +.gov dot gov +.edu dot edu +.uk dot UK +.fr dot FR +.net dot net +.br dot BR +.in dot IN +.ru dot RU +.de dot DE +.it dot IT +.jpg dot jpeg \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5e8973f415334d3a72cf6e49fecf84a803ed2ecc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/symbol.tsv @@ -0,0 +1,19 @@ +. dot +- dash +_ underscore +! exclamation mark +# number sign +$ dollar sign +% percent +& ampersand +' quote +* asterisk ++ plus +/ slash += equal sign +? question mark +^ circumflex +` right single quote +| vertical bar +~ tilde +, comma \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/words.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/words.tsv new file mode 100644 index 0000000000000000000000000000000000000000..784c17d49be9fb5e390ea94d739ebd46baaf7620 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/electronic/words.tsv @@ -0,0 +1,16 @@ +drive +sim +early +access +program +rtx RTX +developer +basepod BASEPOD +cuda CUDA +cv +enterprise +services +nvidia NVIDIA +dgx DGX +pro +help diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/math_operation.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/math_operation.tsv new file mode 100644 index 0000000000000000000000000000000000000000..21e72fd0c40556748d53abc4dbd565300e18aa60 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/math_operation.tsv @@ -0,0 +1,8 @@ ++ plus +- minus +/ divided +÷ divided +: divided +× times +* times +· times \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6a3ad687d070acdd13d6693257031f5b6b3382f7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit.tsv @@ -0,0 +1,135 @@ +amu atomic mass unit +bar bar +° degree +º degree +°c degree Celsius +°C degree Celsius +ºc degree Celsius +ºC degree Celsius +℃ degree Celsius +cm2 square centimeter +cm² square centimeter +cm3 cubic centimeter +cm³ cubic centimeter +cm centimeter +cwt hundredweight +db decibel +dm3 cubic decimeter +dm³ cubic decimeter +dm decimeter +ds decisecond +°f degree Fahrenheit +°F degree Fahrenheit +℉ degree Fahrenheit +ft foot +ghz gigahertz +gw gigawatt +gwh gigawatt hour +hz hertz +" inch +kbps kilobit per second +kcal kilo calory +kgf kilogram force +kg kilogram +khz kilohertz +km2 square kilometer +km² square kilometer +km3 cubic kilometer +km³ cubic kilometer +km kilometer +kpa kilopascal +kwh kilowatt hour +kw kilowatt +kW kilowatt +lb pound +lbs pound +m2 square meter +m² square meter +m3 cubic meter +m³ cubic meter +mbps megabit per second +mg milligram +mhz megahertz +mi2 square mile +mi² square mile +mi3 cubic mile +mi³ cubic mile +cu mi cubic mile +mi mile +min minute +ml milliliter +mm2 square millimeter +mm² square millimeter +mol mole +mpa megapascal +mph mile per hour +ng nanogram +nm nanometer +ns nanosecond +oz ounce +pa pascal +% percent +rad radian +rpm revolution per minute +sq ft square foot +sq mi square mile +sv sievert +tb terabyte +tj terajoule +tl teraliter +v volt +yd yard +μg microgram +μm micrometer +μs microsecond +ω ohm +atm ATM +au AU +bq BQ +cc CC +cd CD +da DA +eb EB +ev EV +f F +gb GB +g G +gl GL +gpa GPA +gy GY +ha HA +h H +hl HL +hp GP +hs HS +kb KB +kl KL +kn KN +kt KT +kv KV +lm LM +ma MA +mA MA +mb MB +mc MC +mf MF +m M +mm MM +ms MS +mv MV +mw MW +pb PB +pg PG +ps PS +s S +tb TB +tb YB +zb ZB +sec second +min minute +hr hour +d day +wk week +mo month +yr year +svc service \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit_alternatives.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit_alternatives.tsv new file mode 100644 index 0000000000000000000000000000000000000000..77e4057ae70b8d7296606f6e21335e2097dc071b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/measure/unit_alternatives.tsv @@ -0,0 +1,43 @@ +atm atmosphere +bq becquerel +cd candela +da dalton +eb exabyte +f degree Fahrenheit +gb gigabyte +g gram +gl gigaliter +ha hectare +h hour +hl hectoliter +hp horsepower +hp horsepower +kb kilobit +kb kilobyte +ma megaampere +mA megaampere +ma milliampere +mA milliampere +mb megabyte +mc megacoulomb +mf megafarad +m meter +m minute +mm millimeter +mm millimeter +mm millimeter +ms megasecond +ms mega siemens +ms millisecond +mv millivolt +mV millivolt +mw megawatt +mW megawatt +pb petabyte +pg petagram +ps petasecond +s second +tb terabyte +tb terabyte +yb yottabyte +zb zettabyte diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5201efd8a3a2b36e6042b6725974ee5b149ae78d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_major.tsv @@ -0,0 +1,39 @@ +$ dollar +$ us dollar +US$ us dollar +฿ Thai Baht +£ pound +€ euro +₩ won +nzd new zealand dollar +rs rupee +chf swiss franc +dkk danish kroner +fim finnish markka +aed arab emirates dirham +¥ yen +czk czech koruna +mro mauritanian ouguiya +pkr pakistani rupee +crc costa rican colon +hk$ hong kong dollar +npr nepalese rupee +awg aruban florin +nok norwegian kroner +tzs tanzanian shilling +sek swedish kronor +cyp cypriot pound +r real +sar saudi riyal +cve cape verde escudo +rsd serbian dinar +dm german mark +shp saint helena pounds +php philippine peso +cad canadian dollar +ssp south sudanese pound +scr seychelles rupee +mvr maldivian rufiyaa +DH dirham +Dh dirham +Dhs. dirham diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_plural.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_plural.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a2f67b3a3d00d67490514623bac7f815720f10b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_plural.tsv @@ -0,0 +1,4 @@ +$ cents +US$ cents +€ cents +£ pence \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_singular.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_singular.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fe629d87b693de1180ee75c642190687a754e126 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/currency_minor_singular.tsv @@ -0,0 +1,3 @@ +$ cent +€ cent +£ penny \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..40804ca56e887706bcfbc15a967e4064163a71db --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv @@ -0,0 +1,70 @@ +/ea each +/dozen per dozen +/sec per second +/min per minute +/hr per hour +/h per hour +/d per day +/wk per week +/mo per month +/yr per year +/svc per service +/L per liter +/l per liter +/gal per gallon +/gallon per gallon +/kg per kilogram +/g per gram +/lb per pound +/pound per pound +/oz per ounce +/ounce per ounce +/unit per unit +/ft per foot +/foot per foot +/' per foot +/in per inch +/inch per inch +/" per inch +/m per meter +/meter per meter +/yard per yard +/yd per yard +/mile per mile +/mi per mile +/square meter per square meter +/sq meter per square meter +/sq m per square meter +/m² per square meter +/square foot per square foot +/sq foot per square foot +/sq ft per square foot +/ft² per square foot +/cubic foot per cubic foot +/cu foot per cubic foot +/cu ft per cubic foot +/ft³ per cubic foot +/cubic meter per cubic meter +/cu meter per cubic meter +/cu m per cubic meter +/m³ per cubic meter +/visit per visit +/page per page +/pg per page +/usr per user +/user per user +/night per night +/passenger per passenger +/word per word +/wd per word +/GB per gigabyte +/MB per megabyte +/TB per terrabyte +/cycle per cycle +/load per load +/person per person +/task per task +/shift per shift +/project per project +/class per class +/session per session \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name.far b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name.far new file mode 100644 index 0000000000000000000000000000000000000000..ba070eedfebde1cfae5a9efbed1c7f4a73afff65 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name.far @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b71c8a37eeba429c5277970fec247023e2f8eccfd8858dbf526b9108bc957cac +size 325078 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name_au.far b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name_au.far new file mode 100644 index 0000000000000000000000000000000000000000..24ab297b29716769826c37e00c9a7173ce6d79ca Binary files /dev/null and b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/cardinal_number_name_au.far differ diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fa329799db6f086e618c8028829b6d2f4b03e88f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/digit.tsv @@ -0,0 +1,9 @@ +one 1 +two 2 +three 3 +four 4 +five 5 +six 6 +seven 7 +eight 8 +nine 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/fraction.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/fraction.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0085ad53ffaa3db2fe0ba3a0c138f582a401142a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/fraction.tsv @@ -0,0 +1,18 @@ +¼ 1/4 +½ 1/2 +¾ 3/4 +⅐ 1/7 +⅑ 1/9 +⅒ 1/10 +⅓ 1/3 +⅔ 2/3 +⅕ 1/5 +⅖ 2/5 +⅗ 3/5 +⅘ 4/5 +⅙ 1/6 +⅚ 5/6 +⅛ 1/8 +⅜ 3/8 +⅝ 5/8 +⅞ 7/8 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/hundred.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/hundred.tsv new file mode 100644 index 0000000000000000000000000000000000000000..10e2b9645a530cfded1cddc74a8cffbf9ed7395f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/hundred.tsv @@ -0,0 +1 @@ +hundred \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/quantity_abbr.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/quantity_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7dba4f03713c1e05aeafcf64fcc1a62d44865344 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/quantity_abbr.tsv @@ -0,0 +1,10 @@ +M million +MLN million +m million +mln million +B billion +b billion +BN billion +bn billion +K thousand +k thousand \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8e60fa1d3cb3cb8fce06150f33e298bee291462a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/teen.tsv @@ -0,0 +1,10 @@ +ten 10 +eleven 11 +twelve 12 +thirteen 13 +fourteen 14 +fifteen 15 +sixteen 16 +seventeen 17 +eighteen 18 +nineteen 19 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/thousand.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/thousand.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bf30542bb6bc1d32435348cd09be00708993dc7b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/thousand.tsv @@ -0,0 +1,22 @@ +thousand +million +billion +trillion +quadrillion +quintillion +sextillion +septillion +octillion +nonillion +decillion +undecillion +duodecillion +tredecillion +quattuordecillion +quindecillion +sexdecillion +septendecillion +octodecillion +novemdecillion +vigintillion +centillion \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/ty.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/ty.tsv new file mode 100644 index 0000000000000000000000000000000000000000..65f9839fda6de480d5b9601ecbc2989749716906 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/ty.tsv @@ -0,0 +1,8 @@ +twenty 2 +thirty 3 +forty 4 +fifty 5 +sixty 6 +seventy 7 +eighty 8 +ninety 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a1b116c10f09fab957175a55f4e19530db49cb3d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/number/zero.tsv @@ -0,0 +1 @@ +zero 0 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4b7fc24f1606194e7da3b936bb7413eee5723dea --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/digit.tsv @@ -0,0 +1,9 @@ +first one +second two +third three +fourth four +fifth five +sixth sixth +seventh seven +eighth eight +ninth nine \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..496fefc086d195f89f422f015594cdbe17158e1b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/ordinal/teen.tsv @@ -0,0 +1 @@ +twelfth twelve \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/README.md b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/README.md new file mode 100644 index 0000000000000000000000000000000000000000..398e1974f823feed75354215533b84b85fa3918d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/README.md @@ -0,0 +1,20 @@ +`female.tsv` - List of common female names. Copyright (c) January 1991 by Mark Kantrowitz, 4987 names, Version 1.3 (29-MAR-94) +Source: [https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/female.txt](https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/female.txt) + +`male.tsv` - List of common male names. Copyright (c) January 1991 by Mark Kantrowitz, 2940 names, Version 1.3 (29-MAR-94) +Source: [https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/male.txt](https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/male.txt) + +[Corpora Readme.txt](https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/readme.txt): + +You may use the lists of names for any purpose, so long as credit is given +in any published work. You may also redistribute the list if you +provide the recipients with a copy of this README file. The lists are +not in the public domain (I retain the copyright on the lists) but are +freely redistributable. + +If you have any additions to the lists of names, I would appreciate +receiving them. + +My email address is mkant+@cs.cmu.edu. + +Mark Kantrowitz \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/female.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/female.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4e248ce2ab7cd08dd038d5c71d91feb10799d599 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/female.tsv @@ -0,0 +1,4998 @@ +Abagael +Abagail +Abbe +Abbey +Abbi +Abbie +Abby +Abigael +Abigail +Abigale +Abra +Acacia +Ada +Adah +Adaline +Adara +Addie +Addis +Adel +Adela +Adelaide +Adele +Adelice +Adelina +Adelind +Adeline +Adella +Adelle +Adena +Adey +Adi +Adiana +Adina +Adora +Adore +Adoree +Adorne +Adrea +Adria +Adriaens +Adrian +Adriana +Adriane +Adrianna +Adrianne +Adrien +Adriena +Adrienne +Aeriel +Aeriela +Aeriell +Ag +Agace +Agata +Agatha +Agathe +Aggi +Aggie +Aggy +Agna +Agnella +Agnes +Agnese +Agnesse +Agneta +Agnola +Agretha +Aida +Aidan +Aigneis +Aila +Aile +Ailee +Aileen +Ailene +Ailey +Aili +Ailina +Ailyn +Aime +Aimee +Aimil +Aina +Aindrea +Ainslee +Ainsley +Ainslie +Ajay +Alaine +Alameda +Alana +Alanah +Alane +Alanna +Alayne +Alberta +Albertina +Albertine +Albina +Alecia +Aleda +Aleece +Aleecia +Aleen +Alejandra +Alejandrina +Alena +Alene +Alessandra +Aleta +Alethea +Alex +Alexa +Alexandra +Alexandrina +Alexi +Alexia +Alexina +Alexine +Alexis +Alfie +Alfreda +Ali +Alia +Alica +Alice +Alicea +Alicia +Alida +Alidia +Alina +Aline +Alis +Alisa +Alisha +Alison +Alissa +Alisun +Alix +Aliza +Alla +Alleen +Allegra +Allene +Alli +Allianora +Allie +Allina +Allis +Allison +Allissa +Allsun +Ally +Allyce +Allyn +Allys +Allyson +Alma +Almeda +Almeria +Almeta +Almira +Almire +Aloise +Aloisia +Aloysia +Alpa +Alta +Althea +Alvera +Alvina +Alvinia +Alvira +Alyce +Alyda +Alys +Alysa +Alyse +Alysia +Alyson +Alyss +Alyssa +Amabel +Amabelle +Amalea +Amalee +Amaleta +Amalia +Amalie +Amalita +Amalle +Amanda +Amandi +Amandie +Amandy +Amara +Amargo +Amata +Amber +Amberly +Ambrosia +Ambur +Ame +Amelia +Amelie +Amelina +Ameline +Amelita +Ami +Amie +Amity +Ammamaria +Amy +Ana +Anabel +Anabella +Anabelle +Anais +Analiese +Analise +Anallese +Anallise +Anastasia +Anastasie +Anastassia +Anatola +Andee +Andi +Andie +Andra +Andrea +Andreana +Andree +Andrei +Andria +Andriana +Andriette +Andromache +Andromeda +Andy +Anestassia +Anet +Anett +Anetta +Anette +Ange +Angel +Angela +Angele +Angelia +Angelica +Angelika +Angelina +Angeline +Angelique +Angelita +Angelle +Angie +Angil +Angy +Ania +Anica +Anissa +Anita +Anitra +Anja +Anjanette +Anjela +Ann +Ann-Mari +Ann-Marie +Anna +Anna-Diana +Anna-Diane +Anna-Maria +Annabal +Annabel +Annabela +Annabell +Annabella +Annabelle +Annadiana +Annadiane +Annalee +Annalena +Annaliese +Annalisa +Annalise +Annalyse +Annamari +Annamaria +Annamarie +Anne +Anne-Corinne +Anne-Mar +Anne-Marie +Annecorinne +Anneliese +Annelise +Annemarie +Annetta +Annette +Anni +Annice +Annie +Annissa +Annmaria +Annmarie +Annnora +Annora +Anny +Anselma +Ansley +Anstice +Anthe +Anthea +Anthia +Antoinette +Antonella +Antonetta +Antonia +Antonie +Antonietta +Antonina +Anya +Aphrodite +Appolonia +Aprilette +Ara +Arabel +Arabela +Arabele +Arabella +Arabelle +Arda +Ardath +Ardeen +Ardelia +Ardelis +Ardella +Ardelle +Arden +Ardene +Ardenia +Ardine +Ardis +Ardith +Ardra +Ardyce +Ardys +Ardyth +Aretha +Ariadne +Ariana +Arianne +Aridatha +Ariel +Ariela +Ariella +Arielle +Arlana +Arlee +Arleen +Arlen +Arlena +Arlene +Arleta +Arlette +Arleyne +Arlie +Arliene +Arlina +Arlinda +Arline +Arly +Arlyn +Arlyne +Aryn +Ashely +Ashlee +Ashleigh +Ashlen +Ashley +Ashli +Ashlie +Ashly +Asia +Astra +Astrid +Astrix +Atalanta +Athena +Athene +Atlanta +Atlante +Auberta +Aubine +Aubree +Aubrette +Aubrey +Aubrie +Aubry +Audi +Audie +Audra +Audre +Audrey +Audrie +Audry +Audrye +Audy +Augusta +Auguste +Augustina +Augustine +Aura +Aurea +Aurel +Aurelea +Aurelia +Aurelie +Auria +Aurie +Aurilia +Aurlie +Auroora +Aurora +Aurore +Austin +Austina +Austine +Ava +Aveline +Averil +Averyl +Avie +Avis +Aviva +Avivah +Avril +Avrit +Ayn +Bab +Babara +Babette +Babita +Babs +Bambi +Bambie +Bamby +Barb +Barbabra +Barbara +Barbara-Anne +Barbaraanne +Barbe +Barbee +Barbette +Barbey +Barbi +Barbie +Barbra +Barby +Bari +Barrie +Barry +Basia +Bathsheba +Batsheva +Bea +Beatrice +Beatrisa +Beatrix +Beatriz +Beau +Bebe +Becca +Becka +Becki +Beckie +Becky +Bee +Beilul +Beitris +Bekki +Bel +Belia +Belicia +Belinda +Belita +Bell +Bella +Bellamy +Bellanca +Belle +Bellina +Belva +Belvia +Bendite +Benedetta +Benedicta +Benedikta +Benetta +Benita +Benni +Bennie +Benny +Benoite +Berenice +Beret +Berget +Berna +Bernadene +Bernadette +Bernadina +Bernadine +Bernardina +Bernardine +Bernelle +Bernete +Bernetta +Bernette +Berni +Bernice +Bernie +Bernita +Berny +Berri +Berrie +Berry +Bert +Berta +Berte +Bertha +Berthe +Berti +Bertie +Bertina +Bertine +Berty +Beryl +Beryle +Bess +Bessie +Bessy +Beth +Bethanne +Bethany +Bethena +Bethina +Betsey +Betsy +Betta +Bette +Bette-Ann +Betteann +Betteanne +Betti +Bettie +Bettina +Bettine +Betty +Bettye +Beulah +Bev +Beverie +Beverlee +Beverlie +Beverly +Bevvy +Bianca +Bianka +Biddy +Bidget +Bill +Billi +Billie +Billy +Binni +Binnie +Binny +Bird +Birdie +Birgit +Birgitta +Blair +Blaire +Blake +Blakelee +Blakeley +Blanca +Blanch +Blancha +Blanche +Blinni +Blinnie +Blinny +Bliss +Blisse +Blithe +Blondell +Blondelle +Blondie +Blondy +Blythe +Bo +Bobbette +Bobbi +Bobbie +Bobby +Bobette +Bobina +Bobine +Bobinette +Bonita +Bonnee +Bonni +Bonnie +Bonny +Brana +Brandais +Brande +Brandea +Brandi +Brandice +Brandie +Brandise +Brandy +Brea +Breanne +Brear +Bree +Breena +Bren +Brena +Brenda +Brenn +Brenna +Brett +Bria +Briana +Brianna +Brianne +Bride +Bridget +Bridgett +Bridgette +Bridie +Brier +Brietta +Brigid +Brigida +Brigit +Brigitta +Brigitte +Brina +Briney +Briny +Brit +Brita +Britaney +Britani +Briteny +Britney +Britni +Britt +Britta +Brittan +Brittany +Britte +Brittney +Brook +Brooke +Brooks +Brunella +Brunhilda +Brunhilde +Bryana +Bryn +Bryna +Brynn +Brynna +Brynne +Buffy +Bunni +Bunnie +Bunny +Burta +Cabrina +Cacilia +Cacilie +Caitlin +Caitrin +Cal +Calida +Calla +Calley +Calli +Callida +Callie +Cally +Calypso +Cam +Camala +Camel +Camella +Camellia +Cameo +Cami +Camila +Camile +Camilla +Camille +Cammi +Cammie +Cammy +Canada +Candace +Candi +Candice +Candida +Candide +Candie +Candis +Candra +Candy +Cappella +Caprice +Cara +Caralie +Caren +Carena +Caresa +Caressa +Caresse +Carey +Cari +Caria +Carie +Caril +Carilyn +Carin +Carina +Carine +Cariotta +Carissa +Carita +Caritta +Carla +Carlee +Carleen +Carlen +Carlena +Carlene +Carley +Carli +Carlie +Carlin +Carlina +Carline +Carlisle +Carlita +Carlota +Carlotta +Carly +Carlye +Carlyn +Carlynn +Carlynne +Carma +Carmel +Carmela +Carmelia +Carmelina +Carmelita +Carmella +Carmelle +Carmen +Carmina +Carmine +Carmita +Carmon +Caro +Carol +Carol-Jean +Carola +Carolan +Carolann +Carole +Carolee +Caroleen +Carolie +Carolin +Carolina +Caroline +Caroljean +Carolyn +Carolyne +Carolynn +Caron +Carree +Carri +Carrie +Carrissa +Carrol +Carroll +Carry +Cary +Caryl +Caryn +Casandra +Casey +Casi +Casia +Casie +Cass +Cassandra +Cassandre +Cassandry +Cassaundra +Cassey +Cassi +Cassie +Cassondra +Cassy +Cat +Catarina +Cate +Caterina +Catha +Catharina +Catharine +Cathe +Cathee +Catherin +Catherina +Catherine +Cathi +Cathie +Cathleen +Cathlene +Cathrin +Cathrine +Cathryn +Cathy +Cathyleen +Cati +Catie +Catina +Catlaina +Catlee +Catlin +Catrina +Catriona +Caty +Cayla +Cecelia +Cecil +Cecile +Ceciley +Cecilia +Cecilla +Cecily +Ceil +Cele +Celene +Celesta +Celeste +Celestia +Celestina +Celestine +Celestyn +Celestyna +Celia +Celie +Celina +Celinda +Celine +Celinka +Celisse +Celle +Cesya +Chad +Chanda +Chandal +Chandra +Channa +Chantal +Chantalle +Charil +Charin +Charis +Charissa +Charisse +Charita +Charity +Charla +Charlean +Charleen +Charlena +Charlene +Charline +Charlot +Charlott +Charlotta +Charlotte +Charmain +Charmaine +Charmane +Charmian +Charmine +Charmion +Charo +Charyl +Chastity +Chelsae +Chelsea +Chelsey +Chelsie +Chelsy +Cher +Chere +Cherey +Cheri +Cherianne +Cherice +Cherida +Cherie +Cherilyn +Cherilynn +Cherin +Cherise +Cherish +Cherlyn +Cherri +Cherrita +Cherry +Chery +Cherye +Cheryl +Cheslie +Chiarra +Chickie +Chicky +Chiquita +Chloe +Chloette +Chloris +Chris +Chriss +Chrissa +Chrissie +Chrissy +Christa +Christabel +Christabella +Christabelle +Christal +Christalle +Christan +Christean +Christel +Christen +Christi +Christian +Christiana +Christiane +Christie +Christin +Christina +Christine +Christy +Christyna +Chrysa +Chrysler +Chrystal +Chryste +Chrystel +Ciara +Cicely +Cicily +Ciel +Cilka +Cinda +Cindee +Cindelyn +Cinderella +Cindi +Cindie +Cindra +Cindy +Cinnamon +Cissie +Cissy +Clair +Claire +Clara +Clarabelle +Clare +Claresta +Clareta +Claretta +Clarette +Clarey +Clari +Claribel +Clarice +Clarie +Clarinda +Clarine +Clarisa +Clarissa +Clarisse +Clarita +Clary +Claude +Claudelle +Claudetta +Claudette +Claudia +Claudie +Claudina +Claudine +Clea +Clem +Clemence +Clementia +Clementina +Clementine +Clemmie +Clemmy +Cleo +Cleopatra +Clerissa +Cleva +Clio +Clo +Cloe +Cloris +Clotilda +Clovis +Codee +Codi +Codie +Cody +Coleen +Colene +Coletta +Colette +Colleen +Collete +Collette +Collie +Colline +Colly +Con +Concettina +Conchita +Concordia +Conney +Conni +Connie +Conny +Consolata +Constance +Constancia +Constancy +Constanta +Constantia +Constantina +Constantine +Consuela +Consuelo +Cookie +Cora +Corabel +Corabella +Corabelle +Coral +Coralie +Coraline +Coralyn +Cordelia +Cordelie +Cordey +Cordie +Cordula +Cordy +Coreen +Corella +Corena +Corenda +Corene +Coretta +Corette +Corey +Cori +Corie +Corilla +Corina +Corine +Corinna +Corinne +Coriss +Corissa +Corliss +Corly +Cornela +Cornelia +Cornelle +Cornie +Corny +Correna +Correy +Corri +Corrianne +Corrie +Corrina +Corrine +Corrinne +Corry +Cortney +Cory +Cosetta +Cosette +Courtenay +Courtney +Cresa +Cris +Crissie +Crissy +Crista +Cristabel +Cristal +Cristen +Cristi +Cristie +Cristin +Cristina +Cristine +Cristionna +Cristy +Crysta +Crystal +Crystie +Cyb +Cybal +Cybel +Cybelle +Cybil +Cybill +Cyndi +Cyndy +Cynthea +Cynthia +Cynthie +Cynthy +Dacey +Dacia +Dacie +Dacy +Dael +Daffi +Daffie +Daffy +Dafna +Dagmar +Dahlia +Daile +Daisey +Daisi +Daisie +Daisy +Dale +Dalenna +Dalia +Dalila +Dallas +Daloris +Damara +Damaris +Damita +Dana +Danell +Danella +Danelle +Danette +Dani +Dania +Danica +Danice +Daniel +Daniela +Daniele +Daniella +Danielle +Danika +Danila +Danit +Danita +Danna +Danni +Dannie +Danny +Dannye +Danya +Danyelle +Danyette +Daphene +Daphna +Daphne +Dara +Darb +Darbie +Darby +Darcee +Darcey +Darci +Darcie +Darcy +Darda +Dareen +Darell +Darelle +Dari +Daria +Darice +Darla +Darleen +Darlene +Darline +Darryl +Darsey +Darsie +Darya +Daryl +Daryn +Dasha +Dasi +Dasie +Dasya +Datha +Daune +Daveen +Daveta +Davida +Davina +Davine +Davita +Dawn +Dawna +Dayle +Dayna +Dea +Deana +Deane +Deanna +Deanne +Deb +Debbi +Debbie +Debbra +Debby +Debee +Debera +Debi +Debor +Debora +Deborah +Debra +Dede +Dedie +Dedra +Dee +Dee Dee +Deeann +Deeanne +Deedee +Deena +Deerdre +Dehlia +Deidre +Deina +Deirdre +Del +Dela +Delaney +Delcina +Delcine +Delia +Delila +Delilah +Delinda +Dell +Della +Delly +Delora +Delores +Deloria +Deloris +Delphina +Delphine +Delphinia +Demeter +Demetra +Demetria +Demetris +Dena +Deni +Denice +Denise +Denna +Denni +Dennie +Denny +Deny +Denys +Denyse +Deonne +Desaree +Desdemona +Desirae +Desiree +Desiri +Deva +Devan +Devi +Devin +Devina +Devinne +Devon +Devondra +Devonna +Devonne +Devora +Dew +Di +Diahann +Diamond +Dian +Diana +Diandra +Diane +Diane-Marie +Dianemarie +Diann +Dianna +Dianne +Diannne +Didi +Dido +Diena +Dierdre +Dina +Dinah +Dinnie +Dinny +Dion +Dione +Dionis +Dionne +Dita +Dix +Dixie +Dode +Dodi +Dodie +Dody +Doe +Doll +Dolley +Dolli +Dollie +Dolly +Dolora +Dolores +Dolorita +Doloritas +Dominica +Dominique +Dona +Donella +Donelle +Donetta +Donia +Donica +Donielle +Donna +Donnajean +Donnamarie +Donni +Donnie +Donny +Dora +Doralia +Doralin +Doralyn +Doralynn +Doralynne +Dorcas +Dore +Doreen +Dorelia +Dorella +Dorelle +Dorena +Dorene +Doretta +Dorette +Dorey +Dori +Doria +Dorian +Dorice +Dorie +Dorine +Doris +Dorisa +Dorise +Dorit +Dorita +Doro +Dorolice +Dorolisa +Dorotea +Doroteya +Dorothea +Dorothee +Dorothy +Dorree +Dorri +Dorrie +Dorris +Dorry +Dorthea +Dorthy +Dory +Dosi +Dot +Doti +Dotti +Dottie +Dotty +Dove +Drea +Drew +Dulce +Dulcea +Dulci +Dulcia +Dulciana +Dulcie +Dulcine +Dulcinea +Dulcy +Dulsea +Dusty +Dyan +Dyana +Dyane +Dyann +Dyanna +Dyanne +Dyna +Dynah +E'Lane +Eada +Eadie +Eadith +Ealasaid +Eartha +Easter +Eba +Ebba +Ebonee +Ebony +Eda +Eddi +Eddie +Eddy +Ede +Edee +Edeline +Eden +Edi +Edie +Edin +Edita +Edith +Editha +Edithe +Ediva +Edna +Edwina +Edy +Edyth +Edythe +Effie +Eileen +Eilis +Eimile +Eirena +Ekaterina +Elaina +Elaine +Elana +Elane +Elayne +Elberta +Elbertina +Elbertine +Eleanor +Eleanora +Eleanore +Electra +Elena +Elene +Eleni +Elenore +Eleonora +Eleonore +Elfie +Elfreda +Elfrida +Elfrieda +Elga +Elianora +Elianore +Elicia +Elie +Elinor +Elinore +Elisa +Elisabet +Elisabeth +Elisabetta +Elise +Elisha +Elissa +Elita +Eliza +Elizabet +Elizabeth +Elka +Elke +Ella +Elladine +Elle +Ellen +Ellene +Ellette +Elli +Ellie +Ellissa +Elly +Ellyn +Ellynn +Elmira +Elna +Elnora +Elnore +Eloisa +Eloise +Elonore +Elora +Elsa +Elsbeth +Else +Elsey +Elsi +Elsie +Elsinore +Elspeth +Elsy +Elva +Elvera +Elvina +Elvira +Elwina +Elwira +Elyn +Elyse +Elysee +Elysha +Elysia +Elyssa +Em +Ema +Emalee +Emalia +Emanuela +Emelda +Emelia +Emelina +Emeline +Emelita +Emelyne +Emera +Emilee +Emili +Emilia +Emilie +Emiline +Emily +Emlyn +Emlynn +Emlynne +Emma +Emmalee +Emmaline +Emmalyn +Emmalynn +Emmalynne +Emmeline +Emmey +Emmi +Emmie +Emmy +Emmye +Emogene +Emyle +Emylee +Endora +Engracia +Enid +Enrica +Enrichetta +Enrika +Enriqueta +Enya +Eolanda +Eolande +Eran +Erda +Erena +Erica +Ericha +Ericka +Erika +Erin +Erina +Erinn +Erinna +Erma +Ermengarde +Ermentrude +Ermina +Erminia +Erminie +Erna +Ernaline +Ernesta +Ernestine +Ertha +Eryn +Esma +Esmaria +Esme +Esmeralda +Esmerelda +Essa +Essie +Essy +Esta +Estel +Estele +Estell +Estella +Estelle +Ester +Esther +Estrella +Estrellita +Ethel +Ethelda +Ethelin +Ethelind +Etheline +Ethelyn +Ethyl +Etta +Etti +Ettie +Etty +Eudora +Eugenia +Eugenie +Eugine +Eula +Eulalie +Eunice +Euphemia +Eustacia +Eva +Evaleen +Evangelia +Evangelin +Evangelina +Evangeline +Evania +Evanne +Eve +Eveleen +Evelina +Eveline +Evelyn +Evette +Evey +Evie +Evita +Evonne +Evvie +Evvy +Evy +Eyde +Eydie +Fabrianne +Fabrice +Fae +Faina +Faith +Fallon +Fan +Fanchette +Fanchon +Fancie +Fancy +Fanechka +Fania +Fanni +Fannie +Fanny +Fanya +Fara +Farah +Farand +Farica +Farra +Farrah +Farrand +Fatima +Faun +Faunie +Faustina +Faustine +Fawn +Fawna +Fawne +Fawnia +Fay +Faydra +Faye +Fayette +Fayina +Fayre +Fayth +Faythe +Federica +Fedora +Felecia +Felicdad +Felice +Felicia +Felicity +Felicle +Felipa +Felisha +Felita +Feliza +Fenelia +Feodora +Ferdinanda +Ferdinande +Fern +Fernanda +Fernande +Fernandina +Ferne +Fey +Fiann +Fianna +Fidela +Fidelia +Fidelity +Fifi +Fifine +Filia +Filide +Filippa +Fina +Fiona +Fionna +Fionnula +Fiorenze +Fleur +Fleurette +Flo +Flor +Flora +Florance +Flore +Florella +Florence +Florencia +Florentia +Florenza +Florette +Flori +Floria +Florice +Florida +Florie +Florina +Florinda +Floris +Florri +Florrie +Florry +Flory +Flossi +Flossie +Flossy +Flower +Fortuna +Fortune +Fran +France +Francene +Frances +Francesca +Francesmary +Francine +Francis +Francisca +Franciska +Francoise +Francyne +Frank +Frankie +Franky +Franni +Frannie +Franny +Frayda +Fred +Freda +Freddi +Freddie +Freddy +Fredelia +Frederica +Fredericka +Fredi +Fredia +Fredra +Fredrika +Freida +Frieda +Friederike +Fulvia +Gabbey +Gabbi +Gabbie +Gabey +Gabi +Gabie +Gabriel +Gabriela +Gabriell +Gabriella +Gabrielle +Gabriellia +Gabrila +Gaby +Gae +Gael +Gail +Gale +Gale +Galina +Garland +Garnet +Garnette +Gates +Gavra +Gavrielle +Gay +Gayla +Gayle +Gayleen +Gaylene +Gaynor +Gelya +Gen +Gena +Gene +Geneva +Genevieve +Genevra +Genia +Genna +Genni +Gennie +Gennifer +Genny +Genovera +Genvieve +George +Georgeanna +Georgeanne +Georgena +Georgeta +Georgetta +Georgette +Georgia +Georgiamay +Georgiana +Georgianna +Georgianne +Georgie +Georgina +Georgine +Gera +Geralda +Geraldina +Geraldine +Gerda +Gerhardine +Geri +Gerianna +Gerianne +Gerladina +Germain +Germaine +Germana +Gerri +Gerrie +Gerrilee +Gerry +Gert +Gerta +Gerti +Gertie +Gertrud +Gertruda +Gertrude +Gertrudis +Gerty +Giacinta +Giana +Gianina +Gianna +Gigi +Gilberta +Gilberte +Gilbertina +Gilbertine +Gilda +Gill +Gillan +Gilli +Gillian +Gillie +Gilligan +Gilly +Gina +Ginelle +Ginevra +Ginger +Ginni +Ginnie +Ginnifer +Ginny +Giorgia +Giovanna +Gipsy +Giralda +Gisela +Gisele +Gisella +Giselle +Gizela +Glad +Gladi +Gladis +Gladys +Gleda +Glen +Glenda +Glenine +Glenn +Glenna +Glennie +Glennis +Glori +Gloria +Gloriana +Gloriane +Glorianna +Glory +Glyn +Glynda +Glynis +Glynnis +Godiva +Golda +Goldarina +Goldi +Goldia +Goldie +Goldina +Goldy +Grace +Gracia +Gracie +Grata +Gratia +Gratiana +Gray +Grayce +Grazia +Gredel +Greer +Greta +Gretal +Gretchen +Grete +Gretel +Grethel +Gretna +Gretta +Grier +Griselda +Grissel +Guendolen +Guenevere +Guenna +Guglielma +Gui +Guillema +Guillemette +Guinevere +Guinna +Gunilla +Gunvor +Gus +Gusella +Gussi +Gussie +Gussy +Gusta +Gusti +Gustie +Gusty +Gwen +Gwendolen +Gwendolin +Gwendolyn +Gweneth +Gwenette +Gwenn +Gwenneth +Gwenni +Gwennie +Gwenny +Gwenora +Gwenore +Gwyn +Gwyneth +Gwynne +Gypsy +Hadria +Hailee +Haily +Haleigh +Halette +Haley +Hali +Halie +Halimeda +Halley +Halli +Hallie +Hally +Hana +Hanna +Hannah +Hanni +Hannibal +Hannie +Hannis +Hanny +Happy +Harlene +Harley +Harli +Harlie +Harmonia +Harmonie +Harmony +Harri +Harrie +Harriet +Harriett +Harrietta +Harriette +Harriot +Harriott +Hatti +Hattie +Hatty +Havivah +Hayley +Hazel +Heath +Heather +Heda +Hedda +Heddi +Heddie +Hedi +Hedvig +Hedwig +Hedy +Heida +Heide +Heidi +Heidie +Helaina +Helaine +Helen +Helen-Elizabeth +Helena +Helene +Helga +Helge +Helise +Hellene +Helli +Heloise +Helsa +Helyn +Hendrika +Henka +Henrie +Henrieta +Henrietta +Henriette +Henryetta +Hephzibah +Hermia +Hermina +Hermine +Herminia +Hermione +Herta +Hertha +Hester +Hesther +Hestia +Hetti +Hettie +Hetty +Hilarie +Hilary +Hilda +Hildagard +Hildagarde +Hilde +Hildegaard +Hildegarde +Hildy +Hillary +Hilliary +Hinda +Holley +Holli +Hollie +Holly +Holly-Anne +Hollyanne +Honey +Honor +Honoria +Hope +Horatia +Hortense +Hortensia +Hulda +Hyacinth +Hyacintha +Hyacinthe +Hyacinthia +Hyacinthie +Hynda +Ianthe +Ibbie +Ibby +Ida +Idalia +Idalina +Idaline +Idell +Idelle +Idette +Ike +Ikey +Ilana +Ileana +Ileane +Ilene +Ilise +Ilka +Illa +Ilona +Ilsa +Ilse +Ilysa +Ilyse +Ilyssa +Imelda +Imogen +Imogene +Imojean +Ina +Inci +Indira +Ines +Inesita +Inessa +Inez +Inga +Ingaberg +Ingaborg +Inge +Ingeberg +Ingeborg +Inger +Ingrid +Ingunna +Inna +Ioana +Iolande +Iolanthe +Iona +Iormina +Ira +Irena +Irene +Irina +Iris +Irita +Irma +Isa +Isabeau +Isabel +Isabelita +Isabella +Isabelle +Isador +Isadora +Isadore +Isahella +Iseabal +Isidora +Isis +Isobel +Issi +Issie +Issy +Ivett +Ivette +Ivie +Ivonne +Ivory +Ivy +Izabel +Izzi +Jacenta +Jacinda +Jacinta +Jacintha +Jacinthe +Jackelyn +Jacki +Jackie +Jacklin +Jacklyn +Jackquelin +Jackqueline +Jacky +Jaclin +Jaclyn +Jacquelin +Jacqueline +Jacquelyn +Jacquelynn +Jacquenetta +Jacquenette +Jacquetta +Jacquette +Jacqui +Jacquie +Jacynth +Jada +Jade +Jaime +Jaimie +Jaine +Jaleh +Jami +Jamie +Jamima +Jammie +Jan +Jana +Janaya +Janaye +Jandy +Jane +Janean +Janeczka +Janeen +Janel +Janela +Janella +Janelle +Janene +Janenna +Janessa +Janet +Janeta +Janetta +Janette +Janeva +Janey +Jania +Janice +Janie +Janifer +Janina +Janine +Janis +Janith +Janka +Janna +Jannel +Jannelle +Janot +Jany +Jaquelin +Jaquelyn +Jaquenetta +Jaquenette +Jaquith +Jasmin +Jasmina +Jasmine +Jayme +Jaymee +Jayne +Jaynell +Jazmin +Jean +Jeana +Jeane +Jeanelle +Jeanette +Jeanie +Jeanine +Jeanna +Jeanne +Jeannette +Jeannie +Jeannine +Jehanna +Jelene +Jemie +Jemima +Jemimah +Jemmie +Jemmy +Jen +Jena +Jenda +Jenelle +Jenette +Jeni +Jenica +Jeniece +Jenifer +Jeniffer +Jenilee +Jenine +Jenn +Jenna +Jennee +Jennette +Jenni +Jennica +Jennie +Jennifer +Jennilee +Jennine +Jenny +Jeraldine +Jeralee +Jere +Jeri +Jermaine +Jerrie +Jerrilee +Jerrilyn +Jerrine +Jerry +Jerrylee +Jess +Jessa +Jessalin +Jessalyn +Jessamine +Jessamyn +Jesse +Jesselyn +Jessi +Jessica +Jessie +Jessika +Jessy +Jewel +Jewell +Jewelle +Jill +Jillana +Jillane +Jillayne +Jilleen +Jillene +Jilli +Jillian +Jillie +Jilly +Jinny +Jo +Jo Ann +Jo-Ann +Jo-Anne +JoAnn +JoAnne +Joan +Joana +Joane +Joanie +Joann +Joanna +Joanne +Joannes +Jobey +Jobi +Jobie +Jobina +Joby +Jobye +Jobyna +Jocelin +Joceline +Jocelyn +Jocelyne +Jodee +Jodi +Jodie +Jody +Joela +Joelie +Joell +Joella +Joelle +Joellen +Joelly +Joellyn +Joelynn +Joete +Joey +Johanna +Johannah +Johnette +Johnna +Joice +Jojo +Jolee +Joleen +Jolene +Joletta +Joli +Jolie +Joline +Joly +Jolyn +Jolynn +Jonell +Joni +Jonie +Jonis +Jordain +Jordan +Jordana +Jordanna +Jorey +Jori +Jorie +Jorrie +Jorry +Joscelin +Josee +Josefa +Josefina +Joselyn +Josepha +Josephina +Josephine +Josey +Josi +Josie +Joslyn +Josselyn +Josy +Jourdan +Joy +Joya +Joyan +Joyann +Joyce +Joycelin +Joye +Joyous +Juana +Juanita +Jude +Judi +Judie +Judith +Juditha +Judy +Judye +Julee +Juli +Julia +Juliana +Juliane +Juliann +Julianna +Julianne +Julie +Julienne +Juliet +Julieta +Julietta +Juliette +Julina +Juline +Julissa +Julita +Junette +Junia +Junie +Junina +Justin +Justina +Justine +Jyoti +Kacey +Kacie +Kacy +Kai +Kaia +Kaila +Kaile +Kailey +Kaitlin +Kaitlyn +Kaitlynn +Kaja +Kakalina +Kala +Kaleena +Kali +Kalie +Kalila +Kalina +Kalinda +Kalindi +Kalli +Kally +Kameko +Kamila +Kamilah +Kamillah +Kandace +Kandy +Kania +Kanya +Kara +Kara-Lynn +Karalee +Karalynn +Kare +Karee +Karel +Karen +Karena +Kari +Karia +Karie +Karil +Karilynn +Karin +Karina +Karine +Kariotta +Karisa +Karissa +Karita +Karla +Karlee +Karleen +Karlen +Karlene +Karlie +Karlotta +Karlotte +Karly +Karlyn +Karmen +Karna +Karol +Karola +Karole +Karolina +Karoline +Karoly +Karon +Karrah +Karrie +Karry +Kary +Karyl +Karylin +Karyn +Kasey +Kass +Kassandra +Kassey +Kassi +Kassia +Kassie +Kaster +Kat +Kata +Katalin +Kate +Katee +Katerina +Katerine +Katey +Kath +Katha +Katharina +Katharine +Katharyn +Kathe +Katheleen +Katherina +Katherine +Katheryn +Kathi +Kathie +Kathleen +Kathlene +Kathlin +Kathrine +Kathryn +Kathryne +Kathy +Kathye +Kati +Katie +Katina +Katine +Katinka +Katleen +Katlin +Katrina +Katrine +Katrinka +Katti +Kattie +Katuscha +Katusha +Katy +Katya +Kay +Kaycee +Kaye +Kayla +Kayle +Kaylee +Kayley +Kaylil +Kaylyn +Kee +Keeley +Keelia +Keely +Kelcey +Kelci +Kelcie +Kelcy +Kelila +Kellen +Kelley +Kelli +Kellia +Kellie +Kellina +Kellsie +Kelly +Kellyann +Kelsey +Kelsi +Kelsy +Kendra +Kendre +Kenna +Keren +Keri +Keriann +Kerianne +Kerri +Kerrie +Kerrill +Kerrin +Kerry +Kerstin +Kesley +Keslie +Kessia +Kessiah +Ketti +Kettie +Ketty +Kevina +Kevyn +Ki +Kia +Kiah +Kial +Kiele +Kiersten +Kikelia +Kiley +Kim +Kimberlee +Kimberley +Kimberli +Kimberly +Kimberlyn +Kimbra +Kimmi +Kimmie +Kimmy +Kinna +Kip +Kipp +Kippie +Kippy +Kira +Kirbee +Kirbie +Kirby +Kiri +Kirsten +Kirsteni +Kirsti +Kirstie +Kirstin +Kirstyn +Kissee +Kissiah +Kissie +Kit +Kitti +Kittie +Kitty +Kizzee +Kizzie +Klara +Klarika +Klarrisa +Konstance +Konstanze +Koo +Kora +Koral +Koralle +Kordula +Kore +Korella +Koren +Koressa +Kori +Korie +Korney +Korrie +Korry +Kourtney +Kris +Krissie +Krissy +Krista +Kristal +Kristan +Kriste +Kristel +Kristen +Kristi +Kristien +Kristin +Kristina +Kristine +Kristy +Kristyn +Krysta +Krystal +Krystalle +Krystle +Krystyna +Kyla +Kyle +Kylen +Kylie +Kylila +Kylynn +Kym +Kynthia +Kyrstin +La +Lacee +Lacey +Lacie +Lacy +Ladonna +Laetitia +Laila +Laina +Lainey +Lamb +Lana +Lane +Lanette +Laney +Lani +Lanie +Lanita +Lanna +Lanni +Lanny +Lara +Laraine +Lari +Larina +Larine +Larisa +Larissa +Lark +Laryssa +Latashia +Latia +Latisha +Latrena +Latrina +Laura +Lauraine +Laural +Lauralee +Laure +Lauree +Laureen +Laurel +Laurella +Lauren +Laurena +Laurene +Lauretta +Laurette +Lauri +Laurianne +Laurice +Laurie +Lauryn +Lavena +Laverna +Laverne +Lavina +Lavinia +Lavinie +Layla +Layne +Layney +Lea +Leah +Leandra +Leann +Leanna +Leanne +Leanor +Leanora +Lebbie +Leda +Lee +LeeAnn +Leeann +Leeanne +Leela +Leelah +Leena +Leesa +Leese +Legra +Leia +Leiah +Leigh +Leigha +Leila +Leilah +Leisha +Lela +Lelah +Leland +Lelia +Lena +Lenee +Lenette +Lenka +Lenna +Lenora +Lenore +Leodora +Leoine +Leola +Leoline +Leona +Leonanie +Leone +Leonelle +Leonie +Leonora +Leonore +Leontine +Leontyne +Leora +Leorah +Leshia +Lesley +Lesli +Leslie +Lesly +Lesya +Leta +Lethia +Leticia +Letisha +Letitia +Letta +Letti +Lettie +Letty +Leyla +Lezlie +Lia +Lian +Liana +Liane +Lianna +Lianne +Lib +Libbey +Libbi +Libbie +Libby +Licha +Lida +Lidia +Lil +Lila +Lilah +Lilas +Lilia +Lilian +Liliane +Lilias +Lilith +Lilla +Lilli +Lillian +Lillis +Lilllie +Lilly +Lily +Lilyan +Lin +Lina +Lind +Linda +Lindi +Lindie +Lindsay +Lindsey +Lindsy +Lindy +Linea +Linell +Linet +Linette +Linn +Linnea +Linnell +Linnet +Linnie +Linzy +Liora +Liorah +Lira +Lisa +Lisabeth +Lisandra +Lisbeth +Lise +Lisetta +Lisette +Lisha +Lishe +Lissa +Lissi +Lissie +Lissy +Lita +Liuka +Livia +Liz +Liza +Lizabeth +Lizbeth +Lizette +Lizzie +Lizzy +Loella +Lois +Loise +Lola +Lolande +Loleta +Lolita +Lolly +Lona +Lonee +Loni +Lonna +Lonni +Lonnie +Lora +Lorain +Loraine +Loralee +Loralie +Loralyn +Loree +Loreen +Lorelei +Lorelle +Loren +Lorena +Lorene +Lorenza +Loretta +Lorettalorna +Lorette +Lori +Loria +Lorianna +Lorianne +Lorie +Lorilee +Lorilyn +Lorinda +Lorine +Lorita +Lorna +Lorne +Lorraine +Lorrayne +Lorri +Lorrie +Lorrin +Lorry +Lory +Lotta +Lotte +Lotti +Lottie +Lotty +Lou +Louella +Louisa +Louise +Louisette +Love +Luana +Luanna +Luce +Luci +Lucia +Luciana +Lucie +Lucienne +Lucila +Lucilia +Lucille +Lucina +Lucinda +Lucine +Lucita +Lucky +Lucretia +Lucy +Luella +Luelle +Luisa +Luise +Lula +Lulita +Lulu +Luna +Lura +Lurette +Lurleen +Lurlene +Lurline +Lusa +Lust +Lyda +Lydia +Lydie +Lyn +Lynda +Lynde +Lyndel +Lyndell +Lyndsay +Lyndsey +Lyndsie +Lyndy +Lynea +Lynelle +Lynett +Lynette +Lynn +Lynna +Lynne +Lynnea +Lynnell +Lynnelle +Lynnet +Lynnett +Lynnette +Lynsey +Lysandra +Lyssa +Mab +Mabel +Mabelle +Mable +Mada +Madalena +Madalyn +Maddalena +Maddi +Maddie +Maddy +Madel +Madelaine +Madeleine +Madelena +Madelene +Madelin +Madelina +Madeline +Madella +Madelle +Madelon +Madelyn +Madge +Madlen +Madlin +Madona +Madonna +Mady +Mae +Maegan +Mag +Magda +Magdaia +Magdalen +Magdalena +Magdalene +Maggee +Maggi +Maggie +Maggy +Magna +Mahala +Mahalia +Maia +Maible +Maiga +Mair +Maire +Mairead +Maisey +Maisie +Mala +Malanie +Malcah +Malena +Malia +Malina +Malinda +Malinde +Malissa +Malissia +Malka +Malkah +Mallissa +Mallorie +Mallory +Malorie +Malory +Malva +Malvina +Malynda +Mame +Mamie +Manda +Mandi +Mandie +Mandy +Manon +Manya +Mara +Marabel +Marcela +Marcelia +Marcella +Marcelle +Marcellina +Marcelline +Marchelle +Marci +Marcia +Marcie +Marcile +Marcille +Marcy +Mareah +Maren +Marena +Maressa +Marga +Margalit +Margalo +Margaret +Margareta +Margarete +Margaretha +Margarethe +Margaretta +Margarette +Margarita +Margaux +Marge +Margeaux +Margery +Marget +Margette +Margi +Margie +Margit +Marglerite +Margo +Margot +Margret +Marguerite +Margurite +Margy +Mari +Maria +Mariam +Marian +Mariana +Mariann +Marianna +Marianne +Maribel +Maribelle +Maribeth +Marice +Maridel +Marie +Marie-Ann +Marie-Jeanne +Marieann +Mariejeanne +Mariel +Mariele +Marielle +Mariellen +Marietta +Mariette +Marigold +Marijo +Marika +Marilee +Marilin +Marillin +Marilyn +Marin +Marina +Marinna +Marion +Mariquilla +Maris +Marisa +Mariska +Marissa +Marit +Marita +Maritsa +Mariya +Marj +Marja +Marje +Marji +Marjie +Marjorie +Marjory +Marjy +Marketa +Marla +Marlane +Marleah +Marlee +Marleen +Marlena +Marlene +Marley +Marlie +Marline +Marlo +Marlyn +Marna +Marne +Marney +Marni +Marnia +Marnie +Marquita +Marrilee +Marris +Marrissa +Marry +Marsha +Marsiella +Marta +Martelle +Martguerita +Martha +Marthe +Marthena +Marti +Martica +Martie +Martina +Martita +Marty +Martynne +Mary +Marya +Maryangelyn +Maryann +Maryanna +Maryanne +Marybelle +Marybeth +Maryellen +Maryjane +Maryjo +Maryl +Marylee +Marylin +Marylinda +Marylou +Marylynne +Maryrose +Marys +Marysa +Masha +Matelda +Mathilda +Mathilde +Matilda +Matilde +Matti +Mattie +Matty +Maud +Maude +Maudie +Maura +Maure +Maureen +Maureene +Maurene +Maurine +Maurise +Maurita +Mavis +Mavra +Max +Maxi +Maxie +Maxine +Maxy +Maya +Maybelle +Mayda +Maye +Mead +Meade +Meagan +Meaghan +Meara +Mechelle +Meg +Megan +Megen +Meggan +Meggi +Meggie +Meggy +Meghan +Meghann +Mehetabel +Mei +Meira +Mel +Mela +Melamie +Melania +Melanie +Melantha +Melany +Melba +Melesa +Melessa +Melicent +Melina +Melinda +Melinde +Melisa +Melisande +Melisandra +Melisenda +Melisent +Melissa +Melisse +Melita +Melitta +Mella +Melli +Mellicent +Mellie +Mellisa +Mellisent +Mellissa +Melloney +Melly +Melodee +Melodie +Melody +Melonie +Melony +Melosa +Melva +Mercedes +Merci +Mercie +Mercy +Meredith +Meredithe +Meridel +Meridith +Meriel +Merilee +Merilyn +Meris +Merissa +Merl +Merla +Merle +Merlina +Merline +Merna +Merola +Merralee +Merridie +Merrie +Merrielle +Merrile +Merrilee +Merrili +Merrill +Merrily +Merry +Mersey +Meryl +Meta +Mia +Micaela +Michaela +Michaelina +Michaeline +Michaella +Michal +Michel +Michele +Michelina +Micheline +Michell +Michelle +Micki +Mickie +Micky +Midge +Mignon +Mignonne +Miguela +Miguelita +Mildred +Mildrid +Milena +Milicent +Milissent +Milka +Milli +Millicent +Millie +Millisent +Milly +Milzie +Mimi +Min +Mina +Minda +Mindy +Minerva +Minetta +Minette +Minna +Minni +Minnie +Minny +Minta +Miquela +Mira +Mirabel +Mirabella +Mirabelle +Miran +Miranda +Mireielle +Mireille +Mirella +Mirelle +Miriam +Mirilla +Mirna +Misha +Missie +Missy +Misti +Misty +Mitra +Mitzi +Mmarianne +Modesta +Modestia +Modestine +Modesty +Moina +Moira +Moll +Mollee +Molli +Mollie +Molly +Mommy +Mona +Monah +Monica +Monika +Monique +Mora +Moreen +Morena +Morgan +Morgana +Morganica +Morganne +Morgen +Moria +Morissa +Morlee +Morna +Moselle +Moya +Moyna +Moyra +Mozelle +Muffin +Mufi +Mufinella +Muire +Mureil +Murial +Muriel +Murielle +Myna +Myra +Myrah +Myranda +Myriam +Myrilla +Myrle +Myrlene +Myrna +Myrta +Myrtia +Myrtice +Myrtie +Myrtle +Nada +Nadean +Nadeen +Nadia +Nadine +Nadiya +Nady +Nadya +Nalani +Nan +Nana +Nananne +Nance +Nancee +Nancey +Nanci +Nancie +Nancy +Nanete +Nanette +Nani +Nanice +Nanine +Nannette +Nanni +Nannie +Nanny +Nanon +Naoma +Naomi +Nara +Nari +Nariko +Nat +Nata +Natala +Natalee +Natalia +Natalie +Natalina +Nataline +Natalya +Natasha +Natassia +Nathalia +Nathalie +Natka +Natty +Neala +Neda +Nedda +Nedi +Neely +Neila +Neile +Neilla +Neille +Nela +Nelia +Nelie +Nell +Nelle +Nelli +Nellie +Nelly +Nena +Nerissa +Nerita +Nert +Nerta +Nerte +Nerti +Nertie +Nerty +Nessa +Nessi +Nessie +Nessy +Nesta +Netta +Netti +Nettie +Nettle +Netty +Nevsa +Neysa +Nichol +Nichole +Nicholle +Nicki +Nickie +Nicky +Nicol +Nicola +Nicole +Nicolea +Nicolette +Nicoli +Nicolina +Nicoline +Nicolle +Nidia +Nike +Niki +Nikki +Nikkie +Nikoletta +Nikolia +Nil +Nina +Ninetta +Ninette +Ninnetta +Ninnette +Ninon +Nisa +Nissa +Nisse +Nissie +Nissy +Nita +Nitin +Nixie +Noami +Noel +Noelani +Noell +Noella +Noelle +Noellyn +Noelyn +Noemi +Nola +Nolana +Nolie +Nollie +Nomi +Nona +Nonah +Noni +Nonie +Nonna +Nonnah +Nora +Norah +Norean +Noreen +Norene +Norina +Norine +Norma +Norri +Norrie +Norry +Nova +Novelia +Nydia +Nyssa +Octavia +Odele +Odelia +Odelinda +Odella +Odelle +Odessa +Odetta +Odette +Odilia +Odille +Ofelia +Ofella +Ofilia +Ola +Olenka +Olga +Olia +Olimpia +Olive +Olivette +Olivia +Olivie +Oliy +Ollie +Olly +Olva +Olwen +Olympe +Olympia +Olympie +Ondrea +Oneida +Onida +Onlea +Oona +Opal +Opalina +Opaline +Ophelia +Ophelie +Oprah +Ora +Oralee +Oralia +Oralie +Oralla +Oralle +Orel +Orelee +Orelia +Orelie +Orella +Orelle +Oreste +Oriana +Orly +Orsa +Orsola +Ortensia +Otha +Othelia +Othella +Othilia +Othilie +Ottilie +Pacifica +Page +Paige +Paloma +Pam +Pamela +Pamelina +Pamella +Pammi +Pammie +Pammy +Pandora +Pansie +Pansy +Paola +Paolina +Parwane +Pat +Patience +Patrica +Patrice +Patricia +Patrizia +Patsy +Patti +Pattie +Patty +Paula +Paula-Grace +Paule +Pauletta +Paulette +Pauli +Paulie +Paulina +Pauline +Paulita +Pauly +Pavia +Pavla +Pearl +Pearla +Pearle +Pearline +Peg +Pegeen +Peggi +Peggie +Peggy +Pen +Penelopa +Penelope +Penni +Pennie +Penny +Pepi +Pepita +Peri +Peria +Perl +Perla +Perle +Perri +Perrine +Perry +Persis +Pet +Peta +Petra +Petrina +Petronella +Petronia +Petronilla +Petronille +Petunia +Phaedra +Phaidra +Phebe +Phedra +Phelia +Phil +Philipa +Philippa +Philippe +Philippine +Philis +Phillida +Phillie +Phillis +Philly +Philomena +Phoebe +Phylis +Phyllida +Phyllis +Phyllys +Phylys +Pia +Pier +Pierette +Pierrette +Pietra +Piper +Pippa +Pippy +Polly +Pollyanna +Pooh +Poppy +Portia +Pris +Prisca +Priscella +Priscilla +Prissie +Pru +Prudence +Prudi +Prudy +Prue +Prunella +Queada +Queenie +Quentin +Querida +Quinn +Quinta +Quintana +Quintilla +Quintina +Rachael +Rachel +Rachele +Rachelle +Rae +Raf +Rafa +Rafaela +Rafaelia +Rafaelita +Ragnhild +Rahal +Rahel +Raina +Raine +Rakel +Ralina +Ramona +Ramonda +Rana +Randa +Randee +Randene +Randi +Randie +Randy +Ranee +Rani +Rania +Ranice +Ranique +Ranna +Raphaela +Raquel +Raquela +Rasia +Rasla +Raven +Ray +Raychel +Raye +Rayna +Raynell +Rayshell +Rea +Reba +Rebbecca +Rebe +Rebeca +Rebecca +Rebecka +Rebeka +Rebekah +Rebekkah +Ree +Reeba +Reena +Reeta +Reeva +Regan +Reggi +Reggie +Regina +Regine +Reiko +Reina +Reine +Remy +Rena +Renae +Renata +Renate +Rene +Renee +Renel +Renell +Renelle +Renie +Rennie +Reta +Retha +Revkah +Rey +Reyna +Rhea +Rheba +Rheta +Rhetta +Rhiamon +Rhianna +Rhianon +Rhoda +Rhodia +Rhodie +Rhody +Rhona +Rhonda +Riane +Riannon +Rianon +Rica +Ricca +Rici +Ricki +Rickie +Ricky +Riki +Rikki +Rina +Risa +Rissa +Rita +Riva +Rivalee +Rivi +Rivkah +Rivy +Roana +Roanna +Roanne +Robbi +Robbie +Robbin +Robby +Robbyn +Robena +Robenia +Roberta +Robin +Robina +Robinet +Robinett +Robinetta +Robinette +Robinia +Roby +Robyn +Roch +Rochell +Rochella +Rochelle +Rochette +Roda +Rodi +Rodie +Rodina +Romola +Romona +Romonda +Romy +Rona +Ronalda +Ronda +Ronica +Ronna +Ronni +Ronnica +Ronnie +Ronny +Roobbie +Rora +Rori +Rorie +Rory +Ros +Rosa +Rosabel +Rosabella +Rosabelle +Rosaleen +Rosalia +Rosalie +Rosalind +Rosalinda +Rosalinde +Rosaline +Rosalyn +Rosalynd +Rosamond +Rosamund +Rosana +Rosanna +Rosanne +Rosario +Rose +Roseann +Roseanna +Roseanne +Roselia +Roselin +Roseline +Rosella +Roselle +Roselyn +Rosemaria +Rosemarie +Rosemary +Rosemonde +Rosene +Rosetta +Rosette +Roshelle +Rosie +Rosina +Rosita +Roslyn +Rosmunda +Rosy +Row +Rowe +Rowena +Roxana +Roxane +Roxanna +Roxanne +Roxi +Roxie +Roxine +Roxy +Roz +Rozalie +Rozalin +Rozamond +Rozanna +Rozanne +Roze +Rozele +Rozella +Rozelle +Rozina +Rubetta +Rubi +Rubia +Rubie +Rubina +Ruby +Ruella +Ruperta +Ruth +Ruthann +Ruthanne +Ruthe +Ruthi +Ruthie +Ruthy +Ryann +Rycca +Saba +Sabina +Sabine +Sabra +Sabrina +Sacha +Sada +Sadella +Sadie +Sal +Sallee +Salli +Sallie +Sally +Sallyann +Sallyanne +Salome +Sam +Samantha +Samara +Samaria +Sammy +Samuela +Samuella +Sande +Sandi +Sandie +Sandra +Sandy +Sandye +Sapphira +Sapphire +Sara +Sara-Ann +Saraann +Sarah +Sarajane +Saree +Sarena +Sarene +Sarette +Sari +Sarina +Sarine +Sarita +Sascha +Sasha +Sashenka +Saudra +Saundra +Savina +Sayre +Scarlet +Scarlett +Scotty +Sean +Seana +Secunda +Seka +Sela +Selena +Selene +Selestina +Selia +Selie +Selina +Selinda +Seline +Sella +Selle +Selma +Sena +Sephira +Serena +Serene +Shaina +Shaine +Shalna +Shalne +Shamit +Shana +Shanda +Shandee +Shandie +Shandra +Shandy +Shane +Shani +Shanie +Shanna +Shannah +Shannen +Shannon +Shanon +Shanta +Shantee +Shara +Sharai +Shari +Sharia +Sharie +Sharity +Sharl +Sharla +Sharleen +Sharlene +Sharline +Sharna +Sharon +Sharona +Sharra +Sharron +Sharyl +Shaun +Shauna +Shawn +Shawna +Shawnee +Shay +Shayla +Shaylah +Shaylyn +Shaylynn +Shayna +Shayne +Shea +Sheba +Sheela +Sheelagh +Sheelah +Sheena +Sheeree +Sheila +Sheila-Kathryn +Sheilah +Sheilakathryn +Shel +Shela +Shelagh +Shelba +Shelbi +Shelby +Shelia +Shell +Shelley +Shelli +Shellie +Shelly +Shena +Sher +Sheree +Sheri +Sherie +Sheril +Sherill +Sherilyn +Sherline +Sherri +Sherrie +Sherry +Sherye +Sheryl +Shilpa +Shina +Shir +Shira +Shirah +Shirl +Shirlee +Shirleen +Shirlene +Shirley +Shirline +Shoshana +Shoshanna +Shoshie +Siana +Sianna +Sib +Sibbie +Sibby +Sibeal +Sibel +Sibella +Sibelle +Sibilla +Sibley +Sibyl +Sibylla +Sibylle +Sidoney +Sidonia +Sidonnie +Sigrid +Sile +Sileas +Silva +Silvana +Silvia +Silvie +Simona +Simone +Simonette +Simonne +Sindee +Sinead +Siobhan +Sioux +Siouxie +Sisely +Sisile +Sissie +Sissy +Sofia +Sofie +Solange +Sondra +Sonia +Sonja +Sonni +Sonnie +Sonnnie +Sonny +Sonya +Sophey +Sophi +Sophia +Sophie +Sophronia +Sorcha +Sosanna +Stace +Stacee +Stacey +Staci +Stacia +Stacie +Stacy +Stafani +Star +Starla +Starlene +Starlin +Starr +Stefa +Stefania +Stefanie +Steffane +Steffi +Steffie +Stella +Stepha +Stephana +Stephani +Stephanie +Stephannie +Stephenie +Stephi +Stephie +Stephine +Stesha +Stevana +Stevena +Stoddard +Storey +Storm +Stormi +Stormie +Stormy +Sue +Sue-elle +Suellen +Sukey +Suki +Sula +Sunny +Sunshine +Susan +Susana +Susanetta +Susann +Susanna +Susannah +Susanne +Susette +Susi +Susie +Sussi +Susy +Suzan +Suzann +Suzanna +Suzanne +Suzetta +Suzette +Suzi +Suzie +Suzy +Suzzy +Sybil +Sybila +Sybilla +Sybille +Sybyl +Sydel +Sydelle +Sydney +Sylvia +Sylvie +Tabatha +Tabbatha +Tabbi +Tabbie +Tabbitha +Tabby +Tabina +Tabitha +Taffy +Talia +Tallia +Tallie +Tally +Talya +Talyah +Tamar +Tamara +Tamarah +Tamarra +Tamera +Tami +Tamiko +Tamma +Tammara +Tammi +Tammie +Tammy +Tamra +Tana +Tandi +Tandie +Tandy +Tani +Tania +Tansy +Tanya +Tara +Tarah +Tarra +Tarrah +Taryn +Tasha +Tasia +Tate +Tatiana +Tatiania +Tatum +Tawnya +Tawsha +Teane +Ted +Tedda +Teddi +Teddie +Teddy +Tedi +Tedra +Teena +Tella +Teodora +Tera +Teresa +TeresaAnne +Terese +Teresina +Teresita +Teressa +Teri +Teriann +Terina +Terra +Terri +Terri-Jo +Terrianne +Terrie +Terry +Terrye +Tersina +Teryl +Terza +Tess +Tessa +Tessi +Tessie +Tessy +Thalia +Thea +Theada +Theadora +Theda +Thekla +Thelma +Theo +Theodora +Theodosia +Theresa +Theresa-Marie +Therese +Theresina +Theresita +Theressa +Therine +Thia +Thomasa +Thomasin +Thomasina +Thomasine +Tia +Tiana +Tiena +Tierney +Tiertza +Tiff +Tiffani +Tiffanie +Tiffany +Tiffi +Tiffie +Tiffy +Tilda +Tildi +Tildie +Tildy +Tillie +Tilly +Tim +Timi +Timmi +Timmie +Timmy +Timothea +Tina +Tine +Tiphani +Tiphanie +Tiphany +Tish +Tisha +Tobe +Tobey +Tobi +Tobie +Toby +Tobye +Toinette +Toma +Tomasina +Tomasine +Tomi +Tomiko +Tommi +Tommie +Tommy +Toni +Tonia +Tonie +Tony +Tonya +Tootsie +Torey +Tori +Torie +Torrie +Tory +Tova +Tove +Trace +Tracee +Tracey +Traci +Tracie +Tracy +Trenna +Tresa +Trescha +Tressa +Tricia +Trina +Trish +Trisha +Trista +Trix +Trixi +Trixie +Trixy +Truda +Trude +Trudey +Trudi +Trudie +Trudy +Trula +Tuesday +Twila +Twyla +Tybi +Tybie +Tyne +Ula +Ulla +Ulrica +Ulrika +Ulrike +Umeko +Una +Ursa +Ursala +Ursola +Ursula +Ursulina +Ursuline +Uta +Val +Valaree +Valaria +Vale +Valeda +Valencia +Valene +Valenka +Valentia +Valentina +Valentine +Valera +Valeria +Valerie +Valery +Valerye +Valida +Valina +Valli +Vallie +Vally +Valma +Valry +Van +Vanda +Vanessa +Vania +Vanna +Vanni +Vannie +Vanny +Vanya +Veda +Velma +Velvet +Vena +Venita +Ventura +Venus +Vera +Veradis +Vere +Verena +Verene +Veriee +Verile +Verina +Verine +Verla +Verna +Vernice +Veronica +Veronika +Veronike +Veronique +Vi +Vicki +Vickie +Vicky +Victoria +Vida +Viki +Vikki +Vikkie +Vikky +Vilhelmina +Vilma +Vin +Vina +Vinita +Vinni +Vinnie +Vinny +Viola +Violante +Viole +Violet +Violetta +Violette +Virgie +Virgina +Virginia +Virginie +Vita +Vitia +Vitoria +Vittoria +Viv +Viva +Vivi +Vivia +Vivian +Viviana +Vivianna +Vivianne +Vivie +Vivien +Viviene +Vivienne +Viviyan +Vivyan +Vivyanne +Vonni +Vonnie +Vonny +Wallie +Wallis +Wally +Waly +Wanda +Wandie +Wandis +Waneta +Wenda +Wendeline +Wendi +Wendie +Wendy +Wenona +Wenonah +Whitney +Wileen +Wilhelmina +Wilhelmine +Wilie +Willa +Willabella +Willamina +Willetta +Willette +Willi +Willie +Willow +Willy +Willyt +Wilma +Wilmette +Wilona +Wilone +Wilow +Windy +Wini +Winifred +Winna +Winnah +Winne +Winni +Winnie +Winnifred +Winny +Winona +Winonah +Wren +Wrennie +Wylma +Wynn +Wynne +Wynnie +Wynny +Xaviera +Xena +Xenia +Xylia +Xylina +Yalonda +Yehudit +Yelena +Yetta +Yettie +Yetty +Yevette +Yoko +Yolanda +Yolande +Yolane +Yolanthe +Yonina +Yoshi +Yoshiko +Yovonnda +Yvette +Yvonne +Zabrina +Zahara +Zandra +Zaneta +Zara +Zarah +Zaria +Zarla +Zea +Zelda +Zelma +Zena +Zenia +Zia +Zilvia +Zita +Zitella +Zoe +Zola +Zonda +Zondra +Zonnya +Zora +Zorah +Zorana +Zorina +Zorine +Zsa Zsa +Zsazsa +Zulema +Zuzana +Mikako +Kaari +Gita +Geeta diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/key_word.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/key_word.tsv new file mode 100644 index 0000000000000000000000000000000000000000..861285b439e6d4d08fef5295421123df2972a26b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/key_word.tsv @@ -0,0 +1,6 @@ +chapter +class +part +article +section +paragraph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/male.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/male.tsv new file mode 100644 index 0000000000000000000000000000000000000000..08236aad9b693fa3e28401cec2cc549d7596d7fb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/male.tsv @@ -0,0 +1,2946 @@ +Aamir +Aaron +Abbey +Abbie +Abbot +Abbott +Abby +Abdel +Abdul +Abdulkarim +Abdullah +Abe +Abel +Abelard +Abner +Abraham +Abram +Ace +Adair +Adam +Adams +Addie +Adger +Aditya +Adlai +Adnan +Adolf +Adolfo +Adolph +Adolphe +Adolpho +Adolphus +Adrian +Adrick +Adrien +Agamemnon +Aguinaldo +Aguste +Agustin +Aharon +Ahmad +Ahmed +Ahmet +Ajai +Ajay +Al +Alaa +Alain +Alan +Alasdair +Alastair +Albatros +Albert +Alberto +Albrecht +Alden +Aldis +Aldo +Aldric +Aldrich +Aldus +Aldwin +Alec +Aleck +Alejandro +Aleks +Aleksandrs +Alessandro +Alex +Alexander +Alexei +Alexis +Alf +Alfie +Alfonse +Alfonso +Alfonzo +Alford +Alfred +Alfredo +Algernon +Ali +Alic +Alister +Alix +Allah +Allan +Allen +Alley +Allie +Allin +Allyn +Alonso +Alonzo +Aloysius +Alphonse +Alphonso +Alston +Alton +Alvin +Alwin +Amadeus +Ambros +Ambrose +Ambrosi +Ambrosio +Ambrosius +Amenhotep +Amery +Amory +Amos +Anatol +Anatole +Anatollo +Anatoly +Anders +Andie +Andonis +Andre +Andrea +Andreas +Andrej +Andres +Andrew +Andrey +Andri +Andros +Andrus +Andrzej +Andy +Angel +Angelico +Angelo +Angie +Angus +Ansel +Ansell +Anselm +Anson +Anthony +Antin +Antoine +Anton +Antone +Antoni +Antonin +Antonino +Antonio +Antonius +Antony +Anurag +Apollo +Apostolos +Aram +Archibald +Archibold +Archie +Archon +Archy +Arel +Ari +Arie +Ariel +Aristotle +Arlo +Armand +Armando +Armond +Armstrong +Arne +Arnie +Arnold +Arnoldo +Aron +Arron +Art +Arther +Arthur +Artie +Artur +Arturo +Arvie +Arvin +Arvind +Arvy +Ash +Ashby +Ashish +Ashley +Ashton +Aub +Aube +Aubert +Aubrey +Augie +Augustin +Augustine +Augusto +Augustus +Austen +Austin +Ave +Averell +Averil +Averill +Avery +Avi +Avraham +Avram +Avrom +Axel +Aylmer +Aziz +Bailey +Bailie +Baillie +Baily +Baird +Baldwin +Bancroft +Barbabas +Barclay +Bard +Barde +Barn +Barnabas +Barnabe +Barnaby +Barnard +Barnebas +Barnett +Barney +Barnie +Barny +Baron +Barr +Barret +Barrett +Barri +Barrie +Barris +Barron +Barry +Bart +Bartel +Barth +Barthel +Bartholemy +Bartholomeo +Bartholomeus +Bartholomew +Bartie +Bartlet +Bartlett +Bartolemo +Bartolomei +Bartolomeo +Barton +Barty +Bary +Basil +Batholomew +Baxter +Bay +Bayard +Beale +Bealle +Bear +Bearnard +Beau +Beaufort +Beauregard +Beck +Bela +Ben +Benedict +Bengt +Benito +Benjamen +Benjamin +Benji +Benjie +Benjy +Benn +Bennet +Bennett +Bennie +Benny +Benson +Bentley +Benton +Beowulf +Berchtold +Berk +Berke +Berkeley +Berkie +Berkley +Bernard +Bernardo +Bernd +Bernhard +Bernie +Bert +Bertie +Bertram +Bertrand +Bharat +Biff +Bill +Billie +Billy +Bing +Binky +Bishop +Bjorn +Bjorne +Blaine +Blair +Blake +Blare +Blayne +Bo +Bob +Bobbie +Bobby +Bogart +Bogdan +Boniface +Boris +Boyce +Boyd +Brad +Braden +Bradford +Bradley +Bradly +Brady +Brandon +Brandy +Brant +Brendan +Brent +Bret +Brett +Brewer +Brewster +Brian +Brice +Briggs +Brinkley +Britt +Brock +Broddie +Broddy +Broderic +Broderick +Brodie +Brody +Bronson +Brook +Brooke +Brooks +Bruce +Bruno +Bryan +Bryant +Bryce +Bryn +Bryon +Bubba +Buck +Bucky +Bud +Buddy +Burgess +Burke +Burl +Burnaby +Burt +Burton +Buster +Butch +Butler +Byram +Byron +Caesar +Cain +Cal +Caldwell +Caleb +Calhoun +Calvin +Cam +Cameron +Cammy +Carey +Carl +Carleigh +Carlie +Carlin +Carlo +Carlos +Carlton +Carlyle +Carmine +Carroll +Carson +Carsten +Carter +Cary +Caryl +Case +Casey +Caspar +Casper +Cass +Cat +Cecil +Cesar +Chad +Chadd +Chaddie +Chaddy +Chadwick +Chaim +Chalmers +Chan +Chance +Chancey +Chanderjit +Chandler +Chane +Chariot +Charles +Charleton +Charley +Charlie +Charlton +Chas +Chase +Chaunce +Chauncey +Che +Chelton +Chen +Chester +Cheston +Chet +Chev +Chevalier +Chevy +Chip +Chris +Chrissy +Christ +Christian +Christiano +Christie +Christof +Christofer +Christoph +Christophe +Christopher +Christorpher +Christos +Christy +Chrisy +Chuck +Churchill +Clair +Claire +Clancy +Clarance +Clare +Clarence +Clark +Clarke +Claude +Claudio +Claudius +Claus +Clay +Clayborn +Clayborne +Claybourne +Clayton +Cleland +Clem +Clemens +Clement +Clemente +Clemmie +Cletus +Cleveland +Cliff +Clifford +Clifton +Clint +Clinten +Clinton +Clive +Clyde +Cob +Cobb +Cobbie +Cobby +Cody +Colbert +Cole +Coleman +Colin +Collin +Collins +Conan +Connie +Connolly +Connor +Conrad +Conroy +Constantin +Constantine +Constantinos +Conway +Cooper +Corbin +Corby +Corey +Corky +Cornelius +Cornellis +Corrie +Cortese +Corwin +Cory +Cosmo +Costa +Courtney +Craig +Crawford +Creighton +Cris +Cristopher +Curt +Curtice +Curtis +Cy +Cyril +Cyrill +Cyrille +Cyrillus +Cyrus +Dabney +Daffy +Dale +Dallas +Dalton +Damian +Damien +Damon +Dan +Dana +Dane +Dani +Danie +Daniel +Dannie +Danny +Dante +Darby +Darcy +Daren +Darian +Darien +Darin +Dario +Darius +Darrel +Darrell +Darren +Darrick +Darrin +Darryl +Darth +Darwin +Daryl +Daryle +Dave +Davey +David +Davidde +Davide +Davidson +Davie +Davin +Davis +Davon +Davoud +Davy +Dawson +Dean +Deane +Del +Delbert +Dell +Delmar +Demetre +Demetri +Demetris +Demetrius +Demosthenis +Denis +Dennie +Dennis +Denny +Derby +Derek +Derick +Derk +Derrek +Derrick +Derrin +Derrol +Derron +Deryl +Desmond +Desmund +Devin +Devon +Dewey +Dewitt +Dexter +Dick +Dickey +Dickie +Diego +Dieter +Dietrich +Dillon +Dimitri +Dimitrios +Dimitris +Dimitrou +Dimitry +Dino +Dion +Dionis +Dionysus +Dirk +Dmitri +Dom +Domenic +Domenico +Dominic +Dominick +Dominique +Don +Donal +Donald +Donn +Donnie +Donny +Donovan +Dorian +Dory +Doug +Douggie +Dougie +Douglas +Douglass +Douglis +Dov +Doyle +Drake +Drew +Dru +Dryke +Duane +Dudley +Duffie +Duffy +Dugan +Duke +Dunc +Duncan +Dunstan +Durand +Durant +Durante +Durward +Dustin +Dwain +Dwaine +Dwane +Dwayne +Dwight +Dylan +Dyson +Earl +Earle +Easton +Eben +Ebeneser +Ebenezer +Eberhard +Ed +Eddie +Eddy +Edgar +Edgardo +Edie +Edmond +Edmund +Edouard +Edsel +Eduard +Eduardo +Edward +Edwin +Efram +Egbert +Ehud +Elbert +Elden +Eldon +Eli +Elias +Elihu +Elijah +Eliot +Eliott +Elisha +Elliot +Elliott +Ellis +Ellsworth +Ellwood +Elmer +Elmore +Elnar +Elric +Elroy +Elton +Elvin +Elvis +Elwin +Elwood +Elwyn +Ely +Emanuel +Emerson +Emery +Emil +Emile +Emilio +Emmanuel +Emmery +Emmet +Emmett +Emmit +Emmott +Emmy +Emory +Ender +Engelbart +Engelbert +Englebart +Englebert +Enoch +Enrico +Enrique +Ephraim +Ephram +Ephrayim +Ephrem +Er +Erasmus +Erastus +Erek +Erhard +Erhart +Eric +Erich +Erick +Erik +Erin +Erl +Ernest +Ernesto +Ernie +Ernst +Erny +Errol +Ervin +Erwin +Esau +Esme +Esteban +Ethan +Ethelbert +Ethelred +Etienne +Euclid +Eugen +Eugene +Eustace +Ev +Evan +Evelyn +Everard +Everett +Ewan +Ewart +Ez +Ezechiel +Ezekiel +Ezra +Fabian +Fabio +Fairfax +Farley +Fazeel +Federico +Felice +Felicio +Felipe +Felix +Ferd +Ferdie +Ferdinand +Ferdy +Fergus +Ferguson +Ferinand +Fernando +Fidel +Filbert +Filip +Filipe +Filmore +Finley +Finn +Fitz +Fitzgerald +Flem +Fleming +Flemming +Fletch +Fletcher +Flin +Flinn +Flint +Flipper +Florian +Floyd +Flynn +Fons +Fonsie +Fonz +Fonzie +Forbes +Ford +Forest +Forester +Forrest +Forrester +Forster +Foster +Fowler +Fox +Fran +Francesco +Francis +Francisco +Francois +Frank +Frankie +Franklin +Franklyn +Franky +Frans +Franz +Fraser +Frazier +Fred +Freddie +Freddy +Frederic +Frederich +Frederick +Frederico +Frederik +Fredric +Fredrick +Freeman +Freemon +Fremont +French +Friedric +Friedrich +Friedrick +Fritz +Fulton +Fyodor +Gabe +Gabriel +Gabriele +Gabriell +Gabriello +Gail +Gale +Galen +Gallagher +Gamaliel +Garcia +Garcon +Gardener +Gardiner +Gardner +Garey +Garfield +Garfinkel +Garold +Garp +Garret +Garrett +Garrot +Garrott +Garry +Garth +Garv +Garvey +Garvin +Garvy +Garwin +Garwood +Gary +Gaspar +Gasper +Gaston +Gav +Gaven +Gavin +Gavriel +Gay +Gayle +Gearard +Gene +Geo +Geof +Geoff +Geoffrey +Geoffry +Georg +George +Georges +Georgia +Georgie +Georgy +Gerald +Geraldo +Gerard +Gere +Gerhard +Gerhardt +Geri +Germaine +Gerold +Gerome +Gerrard +Gerri +Gerrit +Gerry +Gershom +Gershon +Giacomo +Gian +Giancarlo +Giavani +Gibb +Gideon +Giff +Giffard +Giffer +Giffie +Gifford +Giffy +Gil +Gilbert +Gilberto +Gilburt +Giles +Gill +Gilles +Ginger +Gino +Giordano +Giorgi +Giorgio +Giovanne +Giovanni +Giraldo +Giraud +Giuseppe +Glen +Glenn +Glynn +Godard +Godart +Goddard +Goddart +Godfree +Godfrey +Godfry +Godwin +Gomer +Gonzales +Gonzalo +Goober +Goose +Gordan +Gordie +Gordon +Grace +Grady +Graehme +Graeme +Graham +Graig +Grant +Granville +Greg +Gregg +Greggory +Gregor +Gregorio +Gregory +Gretchen +Griff +Griffin +Griffith +Griswold +Grove +Grover +Guido +Guillaume +Guillermo +Gunner +Gunter +Gunther +Gus +Gustaf +Gustav +Gustave +Gustavo +Gustavus +Guthrey +Guthrie +Guthry +Guy +Hadleigh +Hadley +Hadrian +Hagan +Hagen +Hailey +Hakeem +Hakim +Hal +Hale +Haleigh +Haley +Hall +Hallam +Halvard +Ham +Hamel +Hamid +Hamil +Hamilton +Hamish +Hamlen +Hamlet +Hamlin +Hammad +Hamnet +Han +Hanan +Hanford +Hank +Hannibal +Hans +Hans-Peter +Hansel +Hanson +Harald +Harcourt +Hari +Harlan +Harland +Harley +Harlin +Harman +Harmon +Harold +Harris +Harrison +Harrold +Harry +Hart +Hartley +Hartwell +Harv +Harvard +Harvey +Harvie +Harwell +Hasheem +Hashim +Haskel +Haskell +Hassan +Hastings +Hasty +Haven +Hayden +Haydon +Hayes +Hayward +Haywood +Hazel +Heath +Heathcliff +Hebert +Hector +Heinrich +Heinz +Helmuth +Henderson +Hendrick +Hendrik +Henri +Henrie +Henrik +Henrique +Henry +Herb +Herbert +Herbie +Herby +Hercule +Hercules +Herculie +Herman +Hermann +Hermon +Hermy +Hernando +Herold +Herrick +Herrmann +Hersch +Herschel +Hersh +Hershel +Herve +Hervey +Hew +Hewe +Hewet +Hewett +Hewie +Hewitt +Heywood +Hezekiah +Higgins +Hilary +Hilbert +Hill +Hillard +Hillary +Hillel +Hillery +Hilliard +Hilton +Hiralal +Hiram +Hiro +Hirsch +Hobart +Hodge +Hogan +Hollis +Holly +Homer +Horace +Horacio +Horatio +Horatius +Horst +Howard +Howie +Hoyt +Hubert +Hudson +Huey +Hugh +Hugo +Humbert +Humphrey +Hunt +Hunter +Huntington +Huntlee +Huntley +Hurley +Husain +Husein +Hussein +Hy +Hyatt +Hyman +Hymie +Iago +Iain +Ian +Ibrahim +Ichabod +Iggie +Iggy +Ignace +Ignacio +Ignacius +Ignatius +Ignaz +Ignazio +Igor +Ike +Ikey +Immanuel +Ingamar +Ingelbert +Ingemar +Inglebert +Ingmar +Ingram +Inigo +Innocent +Ira +Irvin +Irvine +Irving +Irwin +Isa +Isaac +Isaak +Isador +Isadore +Isaiah +Ishmael +Isidore +Ismail +Israel +Istvan +Ivan +Ivor +Izaak +Izak +Izzy +Jabez +Jack +Jackie +Jackson +Jacob +Jacques +Jae +Jaime +Jake +Jakob +James +Jameson +Jamey +Jamie +Jan +Janos +Janus +Jared +Jarrett +Jarvis +Jason +Jasper +Javier +Jay +Jean +Jean-Christophe +Jean-Francois +Jean-Lou +Jean-Luc +Jean-Marc +Jean-Paul +Jean-Pierre +Jeb +Jed +Jedediah +Jef +Jeff +Jefferey +Jefferson +Jeffery +Jeffie +Jeffrey +Jeffry +Jefry +Jehu +Jennings +Jens +Jephthah +Jerald +Jeramie +Jere +Jereme +Jeremiah +Jeremias +Jeremie +Jeremy +Jermain +Jermaine +Jermayne +Jerold +Jerome +Jeromy +Jerri +Jerrie +Jerrold +Jerrome +Jerry +Jervis +Jerzy +Jess +Jesse +Jessee +Jessey +Jessie +Jesus +Jeth +Jethro +Jim +Jimbo +Jimmie +Jimmy +Jo +Joab +Joachim +Joao +Joaquin +Job +Jock +Jodi +Jodie +Jody +Joe +Joel +Joey +Johan +Johann +Johannes +John +John-David +John-Patrick +Johnathan +Johnathon +Johnnie +Johnny +Johny +Jon +Jonah +Jonas +Jonathan +Jonathon +Jonny +Jordan +Jordon +Jordy +Jorge +Jory +Jose +Josef +Joseph +Josephus +Josh +Joshua +Joshuah +Josiah +Jotham +Juan +Juanita +Jud +Judah +Judas +Judd +Jude +Judith +Judson +Judy +Juergen +Jule +Jules +Julian +Julie +Julio +Julius +Justin +Justis +Kaiser +Kaleb +Kalil +Kalle +Kalman +Kalvin +Kam +Kane +Kareem +Karel +Karim +Karl +Karsten +Kaspar +Keefe +Keenan +Keene +Keil +Keith +Kellen +Kelley +Kelly +Kelsey +Kelvin +Kelwin +Ken +Kendal +Kendall +Kendrick +Kenn +Kennedy +Kenneth +Kenny +Kent +Kenton +Kenyon +Kermie +Kermit +Kerry +Kevan +Kevin +Kim +Kimball +Kimmo +Kin +Kincaid +King +Kingsley +Kingsly +Kingston +Kip +Kirby +Kirk +Kit +Klaus +Klee +Knox +Konrad +Konstantin +Kory +Kostas +Kraig +Kris +Krishna +Kristian +Kristopher +Kristos +Kurt +Kurtis +Kyle +Laird +Lamar +Lambert +Lamont +Lance +Lancelot +Lane +Langston +Lanny +Larry +Lars +Laurance +Lauren +Laurence +Laurens +Laurent +Laurie +Lawerence +Lawrence +Lawson +Lawton +Lay +Layton +Lazar +Lazare +Lazaro +Lazarus +Lazlo +Lee +Lefty +Leif +Leigh +Leighton +Leland +Lem +Lemar +Lemmie +Lemmy +Lemuel +Len +Lenard +Lennie +Lenny +Leo +Leon +Leonard +Leonardo +Leonerd +Leonhard +Leonid +Leonidas +Leopold +Leroy +Les +Lesley +Leslie +Lester +Lev +Levi +Levin +Levon +Levy +Lew +Lewis +Lex +Liam +Lin +Lincoln +Lind +Lindsay +Lindsey +Lindy +Linoel +Linus +Lion +Lionel +Lionello +Llewellyn +Lloyd +Locke +Lockwood +Logan +Lon +Lonnie +Lonny +Loren +Lorenzo +Lorne +Lorrie +Lothar +Lou +Louie +Louis +Lovell +Lowell +Lucas +Luce +Lucian +Luciano +Lucien +Lucio +Lucius +Ludvig +Ludwig +Luigi +Luis +Lukas +Luke +Luther +Lyle +Lyn +Lyndon +Lynn +Mac +Mace +Mack +Mackenzie +Maddie +Maddy +Madison +Magnum +Magnus +Mahesh +Mahmoud +Mahmud +Maison +Major +Malcolm +Manfred +Manish +Manny +Manuel +Marc +Marcel +Marcello +Marcellus +Marcelo +Marchall +Marcio +Marco +Marcos +Marcus +Marietta +Marilu +Mario +Marion +Marius +Mark +Marko +Markos +Markus +Marlin +Marlo +Marlon +Marlow +Marlowe +Marmaduke +Marsh +Marshal +Marshall +Mart +Martainn +Marten +Martie +Martin +Martino +Marty +Martyn +Marv +Marve +Marven +Marvin +Marwin +Mason +Mateo +Mathew +Mathias +Matias +Matt +Matteo +Matthaeus +Mattheus +Matthew +Matthias +Matthieu +Matthiew +Matthus +Mattias +Mattie +Matty +Maurice +Mauricio +Maurie +Maurise +Maurits +Mauritz +Maury +Max +Maxfield +Maxie +Maxim +Maximilian +Maximilien +Maxwell +Mayer +Maynard +Maynord +Mayor +Mead +Meade +Meier +Meir +Mel +Melvin +Melvyn +Menard +Mendel +Mendie +Meredeth +Meredith +Merell +Merill +Merle +Merlin +Merrel +Merrick +Merril +Merrill +Merry +Merv +Mervin +Merwin +Meryl +Meyer +Mic +Micah +Michael +Michail +Michal +Michale +Micheal +Micheil +Michel +Michele +Mick +Mickey +Mickie +Micky +Miguel +Mika +Mikael +Mike +Mikel +Mikey +Mikhail +Miles +Millicent +Milo +Milt +Milton +Mischa +Mitch +Mitchael +Mitchel +Mitchell +Moe +Mohamad +Mohamed +Mohammad +Mohammed +Mohan +Moise +Moises +Moishe +Monroe +Montague +Monte +Montgomery +Monty +Moore +Mordecai +Morgan +Morlee +Morley +Morly +Morrie +Morris +Morry +Morse +Mort +Morten +Mortie +Mortimer +Morton +Morty +Mose +Moses +Moshe +Moss +Muffin +Mugsy +Muhammad +Munmro +Munroe +Murdoch +Murdock +Murphy +Murray +Mustafa +Myke +Myles +Mylo +Myron +Nahum +Napoleon +Nat +Natale +Nate +Nathan +Nathanael +Nathanial +Nathaniel +Nathanil +Neal +Neale +Neall +Nealon +Nealson +Nealy +Ned +Neddie +Neddy +Neel +Neil +Nels +Nelsen +Nelson +Nero +Neron +Nester +Nestor +Nev +Nevil +Nevile +Neville +Nevin +Nevins +Newton +Niall +Niccolo +Nicholas +Nichole +Nichols +Nick +Nickey +Nickie +Nickolas +Nicky +Nico +Nicolas +Niels +Nigel +Niki +Nikita +Nikki +Nikolai +Nikos +Niles +Nils +Nilson +Niven +Noach +Noah +Noam +Noble +Noe +Noel +Nolan +Noland +Norbert +Norm +Norman +Normand +Normie +Norris +Northrop +Northrup +Norton +Norwood +Nunzio +Obadiah +Obadias +Oberon +Obie +Octavius +Odell +Odie +Odin +Odysseus +Olaf +Olag +Ole +Oleg +Olin +Oliver +Olivier +Olle +Ollie +Omar +Oral +Oran +Orazio +Orbadiah +Oren +Orin +Orion +Orlando +Orren +Orrin +Orson +Orton +Orville +Osbert +Osborn +Osborne +Osbourn +Osbourne +Oscar +Osgood +Osmond +Osmund +Ossie +Oswald +Oswell +Otes +Othello +Otho +Otis +Otto +Owen +Ozzie +Ozzy +Pablo +Pace +Paco +Paddie +Paddy +Padraig +Page +Paige +Pail +Palmer +Paolo +Park +Parke +Parker +Parnell +Parrnell +Parry +Parsifal +Partha +Pascal +Pascale +Pasquale +Pat +Pate +Patel +Paten +Patin +Paton +Patric +Patrice +Patricio +Patrick +Patrik +Patsy +Pattie +Patty +Paul +Paulo +Pavel +Pearce +Pedro +Peirce +Pembroke +Pen +Penn +Pennie +Penny +Penrod +Pepe +Pepillo +Pepito +Perceval +Percival +Percy +Perry +Pete +Peter +Petey +Petr +Peyter +Peyton +Phil +Philbert +Philip +Phillip +Phillipe +Phillipp +Phineas +Phip +Pierce +Pierre +Pierson +Piet +Pieter +Pietro +Piggy +Pincas +Pinchas +Pincus +Piotr +Pip +Pius +Plato +Pooh +Porter +Poul +Powell +Praneetf +Prasad +Prasun +Prent +Prentice +Prentiss +Prescott +Preston +Price +Prince +Pryce +Puff +Purcell +Putnam +Pyotr +Quent +Quentin +Quiggly +Quigly +Quigman +Quill +Quillan +Quincey +Quincy +Quinlan +Quinn +Quint +Quintin +Quinton +Quintus +Rab +Rabbi +Rabi +Rad +Radcliffe +Rafael +Rafe +Ragnar +Raimund +Rainer +Raj +Rajeev +Raleigh +Ralf +Ralph +Ram +Rameses +Ramesh +Ramon +Ramsay +Ramsey +Rand +Randal +Randall +Randell +Randi +Randie +Randolf +Randolph +Randy +Ransell +Ransom +Raoul +Raphael +Raul +Ravi +Ravil +Rawley +Ray +Raymond +Raymund +Raymundo +Raynard +Rayner +Raynor +Reagan +Red +Redford +Redmond +Reece +Reed +Rees +Reese +Reg +Regan +Regen +Reggie +Reggis +Reggy +Reginald +Reginauld +Reid +Reilly +Reinhard +Reinhold +Rem +Remington +Remus +Renado +Renaldo +Renard +Renato +Renaud +Renault +Rene +Reube +Reuben +Reuven +Rex +Rey +Reynard +Reynold +Reynolds +Reza +Rhett +Ric +Ricard +Ricardo +Riccardo +Rice +Rich +Richard +Richardo +Richie +Richmond +Richy +Rick +Rickard +Rickey +Ricki +Rickie +Ricky +Rik +Rikki +Riley +Rinaldo +Ripley +Ritch +Ritchie +Roarke +Rob +Robb +Robbert +Robbie +Robert +Roberto +Robin +Robinson +Rochester +Rock +Rockwell +Rocky +Rod +Rodd +Roddie +Roddy +Roderic +Roderich +Roderick +Roderigo +Rodge +Rodger +Rodney +Rodolfo +Rodolph +Rodolphe +Rodrick +Rodrigo +Rodrique +Rog +Roger +Rogers +Roice +Roland +Rolando +Rolf +Rolfe +Rolland +Rollin +Rollins +Rollo +Rolph +Romain +Roman +Romeo +Ron +Ronald +Ronen +Roni +Ronnie +Ronny +Roosevelt +Rory +Roscoe +Ross +Roth +Rourke +Rowland +Roy +Royal +Royce +Rube +Ruben +Rubin +Ruby +Rudd +Ruddie +Ruddy +Rudie +Rudiger +Rudolf +Rudolfo +Rudolph +Rudy +Rudyard +Rufe +Rufus +Rupert +Ruperto +Russ +Russel +Russell +Rustie +Rustin +Rusty +Rutger +Rutherford +Rutledge +Rutter +Ryan +Sal +Salem +Salim +Salman +Salmon +Salomo +Salomon +Salomone +Salvador +Salvatore +Salvidor +Sam +Sammie +Sammy +Sampson +Samson +Samuel +Samuele +Sancho +Sander +Sanders +Sanderson +Sandor +Sandro +Sandy +Sanford +Sanson +Sansone +Sarge +Sargent +Sascha +Sasha +Saul +Sauncho +Saunder +Saunders +Saunderson +Saundra +Saw +Sawyer +Sawyere +Sax +Saxe +Saxon +Say +Sayer +Sayers +Sayre +Sayres +Scarface +Schroeder +Schuyler +Scot +Scott +Scotti +Scottie +Scotty +Seamus +Sean +Sebastian +Sebastiano +Sebastien +See +Selby +Selig +Serge +Sergeant +Sergei +Sergent +Sergio +Seth +Seymour +Shadow +Shaine +Shalom +Shamus +Shamshi-Adad +Shanan +Shane +Shannan +Shannon +Shaughn +Shaun +Shaw +Shawn +Shay +Shayne +Shea +Sheff +Sheffie +Sheffield +Sheffy +Shelby +Shelden +Sheldon +Shell +Shelley +Shelton +Shem +Shep +Shepard +Shepherd +Sheppard +Shepperd +Sheridan +Sherlock +Sherlocke +Sherman +Sherwin +Sherwood +Sherwynd +Shimon +Shlomo +Sholom +Shorty +Shurlock +Shurlocke +Shurwood +Si +Sibyl +Sid +Siddhartha +Sidnee +Sidney +Siegfried +Siffre +Sig +Sigfrid +Sigfried +Sigmund +Silas +Silvain +Silvan +Silvano +Silvanus +Silvester +Silvio +Sim +Simeon +Simmonds +Simon +Simone +Sinclair +Sinclare +Sivert +Siward +Sixtus +Skell +Skelly +Skip +Skipp +Skipper +Skippie +Skippy +Skipton +Sky +Skye +Skylar +Skyler +Slade +Slim +Sloan +Sloane +Sly +Smith +Smitty +Socrates +Sol +Sollie +Solly +Solomon +Somerset +Son +Sonnie +Sonny +Sparky +Spence +Spencer +Spense +Spenser +Spike +Spiro +Spiros +Spud +Srinivas +Stacy +Staffard +Stafford +Staford +Stan +Standford +Stanfield +Stanford +Stanislaw +Stanleigh +Stanley +Stanly +Stanton +Stanwood +Stavros +Stearn +Stearne +Stefan +Stefano +Steffen +Stephan +Stephanus +Stephen +Sterling +Stern +Sterne +Steve +Steven +Stevie +Stevy +Stew +Steward +Stewart +Stig +Stillman +Stillmann +Sting +Stinky +Stirling +Stu +Stuart +Sturgis +Sullivan +Sully +Sumner +Sunny +Sutherland +Sutton +Sven +Swen +Syd +Sydney +Sylvan +Sylvester +Tab +Tabb +Tabbie +Tabby +Taber +Tabor +Tad +Tadd +Taddeo +Taddeus +Tadeas +Tailor +Tait +Taite +Talbert +Talbot +Tallie +Tally +Tam +Tamas +Tammie +Tammy +Tan +Tann +Tanner +Tanney +Tannie +Tanny +Tarrance +Tarrant +Tarzan +Tate +Taylor +Teador +Ted +Tedd +Teddie +Teddy +Tedie +Tedman +Tedmund +Tedrick +Temp +Temple +Templeton +Teodoor +Teodor +Teodorico +Teodoro +Terence +Terencio +Terrance +Terrel +Terrell +Terrence +Terri +Terrill +Terry +Thacher +Thad +Thaddeus +Thaddius +Thaddus +Thadeus +Thain +Thaine +Thane +Tharen +Thatch +Thatcher +Thaxter +Thayne +Thebault +Thedric +Thedrick +Theo +Theobald +Theodor +Theodore +Theodoric +Theophyllus +Thibaud +Thibaut +Thom +Thomas +Thor +Thorn +Thorndike +Thornie +Thornton +Thorny +Thorpe +Thorstein +Thorsten +Thorvald +Thurstan +Thurston +Tibold +Tiebold +Tiebout +Tiler +Tim +Timmie +Timmy +Timothee +Timotheus +Timothy +Tirrell +Tito +Titos +Titus +Tobe +Tobiah +Tobias +Tobie +Tobin +Tobit +Toby +Tod +Todd +Toddie +Toddy +Tom +Tomas +Tome +Tomkin +Tomlin +Tommie +Tommy +Tonnie +Tony +Tore +Torey +Torin +Torr +Torrance +Torre +Torrence +Torrey +Torrin +Torry +Town +Towney +Townie +Townsend +Towny +Trace +Tracey +Tracie +Tracy +Traver +Travers +Travis +Tray +Tre +Tremain +Tremaine +Tremayne +Trent +Trenton +Trev +Trevar +Trever +Trevor +Trey +Trip +Tristan +Troy +Truman +Tuck +Tucker +Tuckie +Tucky +Tudor +Tull +Tulley +Tully +Turner +Ty +Tybalt +Tye +Tyler +Tymon +Tymothy +Tynan +Tyrone +Tyrus +Tyson +Udale +Udall +Udell +Ugo +Ulberto +Uli +Ulick +Ulises +Ulric +Ulrich +Ulrick +Ulysses +Umberto +Upton +Urbain +Urban +Urbano +Urbanus +Uri +Uriah +Uriel +Urson +Vachel +Vaclav +Vail +Val +Valdemar +Vale +Valentin +Valentine +Valentinian +Van +Vance +Vasili +Vasilis +Vasily +Vassili +Vassily +Vaughan +Vaughn +Venkat +Verge +Vergil +Vern +Verne +Vernen +Verney +Vernon +Vernor +Vic +Vick +Victor +Vijay +Vilhelm +Vin +Vince +Vincent +Vincents +Vinnie +Vinny +Vinod +Virge +Virgie +Virgil +Virgilio +Vite +Vito +Vlad +Vladamir +Vladimir +Voltaire +Von +Wade +Wadsworth +Wain +Waine +Wainwright +Wait +Waite +Waiter +Wake +Wakefield +Wald +Waldemar +Walden +Waldo +Waldon +Waleed +Walker +Wallace +Wallache +Wallas +Wallie +Wallis +Wally +Walsh +Walt +Walter +Walther +Walton +Wang +Ward +Warde +Warden +Ware +Waring +Warner +Warren +Wash +Washington +Wat +Waverley +Waverly +Way +Waylan +Wayland +Waylen +Waylin +Waylon +Wayne +Web +Webb +Weber +Webster +Weidar +Weider +Welbie +Welby +Welch +Wells +Welsh +Wendall +Wendel +Wendell +Werner +Wes +Wesley +Weslie +West +Westbrook +Westbrooke +Westleigh +Westley +Weston +Weylin +Wheeler +Whit +Whitaker +Whitby +Whitman +Whitney +Whittaker +Wiatt +Wilber +Wilbert +Wilbur +Wilburn +Wilburt +Wilden +Wildon +Wilek +Wiley +Wilfred +Wilfrid +Wilhelm +Will +Willard +Willdon +Willem +Willey +Willi +William +Willie +Willis +Willmott +Willy +Wilmar +Wilmer +Wilson +Wilt +Wilton +Win +Windham +Winfield +Winford +Winfred +Winifield +Winn +Winnie +Winny +Winslow +Winston +Winthrop +Winton +Wit +Witold +Wittie +Witty +Wojciech +Wolfgang +Wolfie +Wolfram +Wolfy +Woochang +Woodie +Woodman +Woodrow +Woody +Worden +Worthington +Worthy +Wright +Wyatan +Wyatt +Wye +Wylie +Wyn +Wyndham +Wynn +Wynton +Xavier +Xenos +Xerxes +Xever +Ximenes +Ximenez +Xymenes +Yaakov +Yacov +Yale +Yanaton +Yance +Yancey +Yancy +Yank +Yankee +Yard +Yardley +Yehudi +Yigal +Yule +Yuri +Yves +Zach +Zacharia +Zachariah +Zacharias +Zacharie +Zachary +Zacherie +Zachery +Zack +Zackariah +Zak +Zalman +Zane +Zared +Zary +Zeb +Zebadiah +Zebedee +Zebulen +Zebulon +Zechariah +Zed +Zedekiah +Zeke +Zelig +Zerk +Zeus +Zippy +Zollie +Zolly +Zorro +Rahul +Shumeet +Vibhu diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/roman_to_spoken.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/roman_to_spoken.tsv new file mode 100644 index 0000000000000000000000000000000000000000..971c90e37b22d78de53c7b6430c7847922ce5e16 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/roman/roman_to_spoken.tsv @@ -0,0 +1,2000 @@ +I one +II two +III three +IV four +V five +VI six +VII seven +VIII eight +IX nine +X ten +XI eleven +XII twelve +XIII thirteen +XIV fourteen +XV fifteen +XVI sixteen +XVII seventeen +XVIII eighteen +XIX nineteen +XX twenty +XXI twenty one +XXII twenty two +XXIII twenty three +XXIV twenty four +XXV twenty five +XXVI twenty six +XXVII twenty seven +XXVIII twenty eight +XXIX twenty nine +XXX thirty +XXXI thirty one +XXXII thirty two +XXXIII thirty three +XXXIV thirty four +XXXV thirty five +XXXVI thirty six +XXXVII thirty seven +XXXVIII thirty eight +XXXIX thirty nine +XL forty +XLI forty one +XLII forty two +XLIII forty three +XLIV forty four +XLV forty five +XLVI forty six +XLVII forty seven +XLVIII forty eight +XLIX forty nine +L fifty +LI fifty one +LII fifty two +LIII fifty three +LIV fifty four +LV fifty five +LVI fifty six +LVII fifty seven +LVIII fifty eight +LIX fifty nine +LX sixty +LXI sixty one +LXII sixty two +LXIII sixty three +LXIV sixty four +LXV sixty five +LXVI sixty six +LXVII sixty seven +LXVIII sixty eight +LXIX sixty nine +LXX seventy +LXXI seventy one +LXXII seventy two +LXXIII seventy three +LXXIV seventy four +LXXV seventy five +LXXVI seventy six +LXXVII seventy seven +LXXVIII seventy eight +LXXIX seventy nine +LXXX eighty +LXXXI eighty one +LXXXII eighty two +LXXXIII eighty three +LXXXIV eighty four +LXXXV eighty five +LXXXVI eighty six +LXXXVII eighty seven +LXXXVIII eighty eight +LXXXIX eighty nine +XC ninety +XCI ninety one +XCII ninety two +XCIII ninety three +XCIV ninety four +XCV ninety five +XCVI ninety six +XCVII ninety seven +XCVIII ninety eight +XCIX ninety nine +C one hundred +CI one hundred one +CII one hundred two +CIII one hundred three +CIV one hundred four +CV one hundred five +CVI one hundred six +CVII one hundred seven +CVIII one hundred eight +CIX one hundred nine +CX one hundred ten +CXI one hundred eleven +CXII one hundred twelve +CXIII one hundred thirteen +CXIV one hundred fourteen +CXV one hundred fifteen +CXVI one hundred sixteen +CXVII one hundred seventeen +CXVIII one hundred eighteen +CXIX one hundred nineteen +CXX one hundred twenty +CXXI one hundred twenty one +CXXII one hundred twenty two +CXXIII one hundred twenty three +CXXIV one hundred twenty four +CXXV one hundred twenty five +CXXVI one hundred twenty six +CXXVII one hundred twenty seven +CXXVIII one hundred twenty eight +CXXIX one hundred twenty nine +CXXX one hundred thirty +CXXXI one hundred thirty one +CXXXII one hundred thirty two +CXXXIII one hundred thirty three +CXXXIV one hundred thirty four +CXXXV one hundred thirty five +CXXXVI one hundred thirty six +CXXXVII one hundred thirty seven +CXXXVIII one hundred thirty eight +CXXXIX one hundred thirty nine +CXL one hundred forty +CXLI one hundred forty one +CXLII one hundred forty two +CXLIII one hundred forty three +CXLIV one hundred forty four +CXLV one hundred forty five +CXLVI one hundred forty six +CXLVII one hundred forty seven +CXLVIII one hundred forty eight +CXLIX one hundred forty nine +CL one hundred fifty +CLI one hundred fifty one +CLII one hundred fifty two +CLIII one hundred fifty three +CLIV one hundred fifty four +CLV one hundred fifty five +CLVI one hundred fifty six +CLVII one hundred fifty seven +CLVIII one hundred fifty eight +CLIX one hundred fifty nine +CLX one hundred sixty +CLXI one hundred sixty one +CLXII one hundred sixty two +CLXIII one hundred sixty three +CLXIV one hundred sixty four +CLXV one hundred sixty five +CLXVI one hundred sixty six +CLXVII one hundred sixty seven +CLXVIII one hundred sixty eight +CLXIX one hundred sixty nine +CLXX one hundred seventy +CLXXI one hundred seventy one +CLXXII one hundred seventy two +CLXXIII one hundred seventy three +CLXXIV one hundred seventy four +CLXXV one hundred seventy five +CLXXVI one hundred seventy six +CLXXVII one hundred seventy seven +CLXXVIII one hundred seventy eight +CLXXIX one hundred seventy nine +CLXXX one hundred eighty +CLXXXI one hundred eighty one +CLXXXII one hundred eighty two +CLXXXIII one hundred eighty three +CLXXXIV one hundred eighty four +CLXXXV one hundred eighty five +CLXXXVI one hundred eighty six +CLXXXVII one hundred eighty seven +CLXXXVIII one hundred eighty eight +CLXXXIX one hundred eighty nine +CXC one hundred ninety +CXCI one hundred ninety one +CXCII one hundred ninety two +CXCIII one hundred ninety three +CXCIV one hundred ninety four +CXCV one hundred ninety five +CXCVI one hundred ninety six +CXCVII one hundred ninety seven +CXCVIII one hundred ninety eight +CXCIX one hundred ninety nine +CC two hundred +CCI two hundred one +CCII two hundred two +CCIII two hundred three +CCIV two hundred four +CCV two hundred five +CCVI two hundred six +CCVII two hundred seven +CCVIII two hundred eight +CCIX two hundred nine +CCX two hundred ten +CCXI two hundred eleven +CCXII two hundred twelve +CCXIII two hundred thirteen +CCXIV two hundred fourteen +CCXV two hundred fifteen +CCXVI two hundred sixteen +CCXVII two hundred seventeen +CCXVIII two hundred eighteen +CCXIX two hundred nineteen +CCXX two hundred twenty +CCXXI two hundred twenty one +CCXXII two hundred twenty two +CCXXIII two hundred twenty three +CCXXIV two hundred twenty four +CCXXV two hundred twenty five +CCXXVI two hundred twenty six +CCXXVII two hundred twenty seven +CCXXVIII two hundred twenty eight +CCXXIX two hundred twenty nine +CCXXX two hundred thirty +CCXXXI two hundred thirty one +CCXXXII two hundred thirty two +CCXXXIII two hundred thirty three +CCXXXIV two hundred thirty four +CCXXXV two hundred thirty five +CCXXXVI two hundred thirty six +CCXXXVII two hundred thirty seven +CCXXXVIII two hundred thirty eight +CCXXXIX two hundred thirty nine +CCXL two hundred forty +CCXLI two hundred forty one +CCXLII two hundred forty two +CCXLIII two hundred forty three +CCXLIV two hundred forty four +CCXLV two hundred forty five +CCXLVI two hundred forty six +CCXLVII two hundred forty seven +CCXLVIII two hundred forty eight +CCXLIX two hundred forty nine +CCL two hundred fifty +CCLI two hundred fifty one +CCLII two hundred fifty two +CCLIII two hundred fifty three +CCLIV two hundred fifty four +CCLV two hundred fifty five +CCLVI two hundred fifty six +CCLVII two hundred fifty seven +CCLVIII two hundred fifty eight +CCLIX two hundred fifty nine +CCLX two hundred sixty +CCLXI two hundred sixty one +CCLXII two hundred sixty two +CCLXIII two hundred sixty three +CCLXIV two hundred sixty four +CCLXV two hundred sixty five +CCLXVI two hundred sixty six +CCLXVII two hundred sixty seven +CCLXVIII two hundred sixty eight +CCLXIX two hundred sixty nine +CCLXX two hundred seventy +CCLXXI two hundred seventy one +CCLXXII two hundred seventy two +CCLXXIII two hundred seventy three +CCLXXIV two hundred seventy four +CCLXXV two hundred seventy five +CCLXXVI two hundred seventy six +CCLXXVII two hundred seventy seven +CCLXXVIII two hundred seventy eight +CCLXXIX two hundred seventy nine +CCLXXX two hundred eighty +CCLXXXI two hundred eighty one +CCLXXXII two hundred eighty two +CCLXXXIII two hundred eighty three +CCLXXXIV two hundred eighty four +CCLXXXV two hundred eighty five +CCLXXXVI two hundred eighty six +CCLXXXVII two hundred eighty seven +CCLXXXVIII two hundred eighty eight +CCLXXXIX two hundred eighty nine +CCXC two hundred ninety +CCXCI two hundred ninety one +CCXCII two hundred ninety two +CCXCIII two hundred ninety three +CCXCIV two hundred ninety four +CCXCV two hundred ninety five +CCXCVI two hundred ninety six +CCXCVII two hundred ninety seven +CCXCVIII two hundred ninety eight +CCXCIX two hundred ninety nine +CCC three hundred +CCCI three hundred one +CCCII three hundred two +CCCIII three hundred three +CCCIV three hundred four +CCCV three hundred five +CCCVI three hundred six +CCCVII three hundred seven +CCCVIII three hundred eight +CCCIX three hundred nine +CCCX three hundred ten +CCCXI three hundred eleven +CCCXII three hundred twelve +CCCXIII three hundred thirteen +CCCXIV three hundred fourteen +CCCXV three hundred fifteen +CCCXVI three hundred sixteen +CCCXVII three hundred seventeen +CCCXVIII three hundred eighteen +CCCXIX three hundred nineteen +CCCXX three hundred twenty +CCCXXI three hundred twenty one +CCCXXII three hundred twenty two +CCCXXIII three hundred twenty three +CCCXXIV three hundred twenty four +CCCXXV three hundred twenty five +CCCXXVI three hundred twenty six +CCCXXVII three hundred twenty seven +CCCXXVIII three hundred twenty eight +CCCXXIX three hundred twenty nine +CCCXXX three hundred thirty +CCCXXXI three hundred thirty one +CCCXXXII three hundred thirty two +CCCXXXIII three hundred thirty three +CCCXXXIV three hundred thirty four +CCCXXXV three hundred thirty five +CCCXXXVI three hundred thirty six +CCCXXXVII three hundred thirty seven +CCCXXXVIII three hundred thirty eight +CCCXXXIX three hundred thirty nine +CCCXL three hundred forty +CCCXLI three hundred forty one +CCCXLII three hundred forty two +CCCXLIII three hundred forty three +CCCXLIV three hundred forty four +CCCXLV three hundred forty five +CCCXLVI three hundred forty six +CCCXLVII three hundred forty seven +CCCXLVIII three hundred forty eight +CCCXLIX three hundred forty nine +CCCL three hundred fifty +CCCLI three hundred fifty one +CCCLII three hundred fifty two +CCCLIII three hundred fifty three +CCCLIV three hundred fifty four +CCCLV three hundred fifty five +CCCLVI three hundred fifty six +CCCLVII three hundred fifty seven +CCCLVIII three hundred fifty eight +CCCLIX three hundred fifty nine +CCCLX three hundred sixty +CCCLXI three hundred sixty one +CCCLXII three hundred sixty two +CCCLXIII three hundred sixty three +CCCLXIV three hundred sixty four +CCCLXV three hundred sixty five +CCCLXVI three hundred sixty six +CCCLXVII three hundred sixty seven +CCCLXVIII three hundred sixty eight +CCCLXIX three hundred sixty nine +CCCLXX three hundred seventy +CCCLXXI three hundred seventy one +CCCLXXII three hundred seventy two +CCCLXXIII three hundred seventy three +CCCLXXIV three hundred seventy four +CCCLXXV three hundred seventy five +CCCLXXVI three hundred seventy six +CCCLXXVII three hundred seventy seven +CCCLXXVIII three hundred seventy eight +CCCLXXIX three hundred seventy nine +CCCLXXX three hundred eighty +CCCLXXXI three hundred eighty one +CCCLXXXII three hundred eighty two +CCCLXXXIII three hundred eighty three +CCCLXXXIV three hundred eighty four +CCCLXXXV three hundred eighty five +CCCLXXXVI three hundred eighty six +CCCLXXXVII three hundred eighty seven +CCCLXXXVIII three hundred eighty eight +CCCLXXXIX three hundred eighty nine +CCCXC three hundred ninety +CCCXCI three hundred ninety one +CCCXCII three hundred ninety two +CCCXCIII three hundred ninety three +CCCXCIV three hundred ninety four +CCCXCV three hundred ninety five +CCCXCVI three hundred ninety six +CCCXCVII three hundred ninety seven +CCCXCVIII three hundred ninety eight +CCCXCIX three hundred ninety nine +CD four hundred +CDI four hundred one +CDII four hundred two +CDIII four hundred three +CDIV four hundred four +CDV four hundred five +CDVI four hundred six +CDVII four hundred seven +CDVIII four hundred eight +CDIX four hundred nine +CDX four hundred ten +CDXI four hundred eleven +CDXII four hundred twelve +CDXIII four hundred thirteen +CDXIV four hundred fourteen +CDXV four hundred fifteen +CDXVI four hundred sixteen +CDXVII four hundred seventeen +CDXVIII four hundred eighteen +CDXIX four hundred nineteen +CDXX four hundred twenty +CDXXI four hundred twenty one +CDXXII four hundred twenty two +CDXXIII four hundred twenty three +CDXXIV four hundred twenty four +CDXXV four hundred twenty five +CDXXVI four hundred twenty six +CDXXVII four hundred twenty seven +CDXXVIII four hundred twenty eight +CDXXIX four hundred twenty nine +CDXXX four hundred thirty +CDXXXI four hundred thirty one +CDXXXII four hundred thirty two +CDXXXIII four hundred thirty three +CDXXXIV four hundred thirty four +CDXXXV four hundred thirty five +CDXXXVI four hundred thirty six +CDXXXVII four hundred thirty seven +CDXXXVIII four hundred thirty eight +CDXXXIX four hundred thirty nine +CDXL four hundred forty +CDXLI four hundred forty one +CDXLII four hundred forty two +CDXLIII four hundred forty three +CDXLIV four hundred forty four +CDXLV four hundred forty five +CDXLVI four hundred forty six +CDXLVII four hundred forty seven +CDXLVIII four hundred forty eight +CDXLIX four hundred forty nine +CDL four hundred fifty +CDLI four hundred fifty one +CDLII four hundred fifty two +CDLIII four hundred fifty three +CDLIV four hundred fifty four +CDLV four hundred fifty five +CDLVI four hundred fifty six +CDLVII four hundred fifty seven +CDLVIII four hundred fifty eight +CDLIX four hundred fifty nine +CDLX four hundred sixty +CDLXI four hundred sixty one +CDLXII four hundred sixty two +CDLXIII four hundred sixty three +CDLXIV four hundred sixty four +CDLXV four hundred sixty five +CDLXVI four hundred sixty six +CDLXVII four hundred sixty seven +CDLXVIII four hundred sixty eight +CDLXIX four hundred sixty nine +CDLXX four hundred seventy +CDLXXI four hundred seventy one +CDLXXII four hundred seventy two +CDLXXIII four hundred seventy three +CDLXXIV four hundred seventy four +CDLXXV four hundred seventy five +CDLXXVI four hundred seventy six +CDLXXVII four hundred seventy seven +CDLXXVIII four hundred seventy eight +CDLXXIX four hundred seventy nine +CDLXXX four hundred eighty +CDLXXXI four hundred eighty one +CDLXXXII four hundred eighty two +CDLXXXIII four hundred eighty three +CDLXXXIV four hundred eighty four +CDLXXXV four hundred eighty five +CDLXXXVI four hundred eighty six +CDLXXXVII four hundred eighty seven +CDLXXXVIII four hundred eighty eight +CDLXXXIX four hundred eighty nine +CDXC four hundred ninety +CDXCI four hundred ninety one +CDXCII four hundred ninety two +CDXCIII four hundred ninety three +CDXCIV four hundred ninety four +CDXCV four hundred ninety five +CDXCVI four hundred ninety six +CDXCVII four hundred ninety seven +CDXCVIII four hundred ninety eight +CDXCIX four hundred ninety nine +D five hundred +DI five hundred one +DII five hundred two +DIII five hundred three +DIV five hundred four +DV five hundred five +DVI five hundred six +DVII five hundred seven +DVIII five hundred eight +DIX five hundred nine +DX five hundred ten +DXI five hundred eleven +DXII five hundred twelve +DXIII five hundred thirteen +DXIV five hundred fourteen +DXV five hundred fifteen +DXVI five hundred sixteen +DXVII five hundred seventeen +DXVIII five hundred eighteen +DXIX five hundred nineteen +DXX five hundred twenty +DXXI five hundred twenty one +DXXII five hundred twenty two +DXXIII five hundred twenty three +DXXIV five hundred twenty four +DXXV five hundred twenty five +DXXVI five hundred twenty six +DXXVII five hundred twenty seven +DXXVIII five hundred twenty eight +DXXIX five hundred twenty nine +DXXX five hundred thirty +DXXXI five hundred thirty one +DXXXII five hundred thirty two +DXXXIII five hundred thirty three +DXXXIV five hundred thirty four +DXXXV five hundred thirty five +DXXXVI five hundred thirty six +DXXXVII five hundred thirty seven +DXXXVIII five hundred thirty eight +DXXXIX five hundred thirty nine +DXL five hundred forty +DXLI five hundred forty one +DXLII five hundred forty two +DXLIII five hundred forty three +DXLIV five hundred forty four +DXLV five hundred forty five +DXLVI five hundred forty six +DXLVII five hundred forty seven +DXLVIII five hundred forty eight +DXLIX five hundred forty nine +DL five hundred fifty +DLI five hundred fifty one +DLII five hundred fifty two +DLIII five hundred fifty three +DLIV five hundred fifty four +DLV five hundred fifty five +DLVI five hundred fifty six +DLVII five hundred fifty seven +DLVIII five hundred fifty eight +DLIX five hundred fifty nine +DLX five hundred sixty +DLXI five hundred sixty one +DLXII five hundred sixty two +DLXIII five hundred sixty three +DLXIV five hundred sixty four +DLXV five hundred sixty five +DLXVI five hundred sixty six +DLXVII five hundred sixty seven +DLXVIII five hundred sixty eight +DLXIX five hundred sixty nine +DLXX five hundred seventy +DLXXI five hundred seventy one +DLXXII five hundred seventy two +DLXXIII five hundred seventy three +DLXXIV five hundred seventy four +DLXXV five hundred seventy five +DLXXVI five hundred seventy six +DLXXVII five hundred seventy seven +DLXXVIII five hundred seventy eight +DLXXIX five hundred seventy nine +DLXXX five hundred eighty +DLXXXI five hundred eighty one +DLXXXII five hundred eighty two +DLXXXIII five hundred eighty three +DLXXXIV five hundred eighty four +DLXXXV five hundred eighty five +DLXXXVI five hundred eighty six +DLXXXVII five hundred eighty seven +DLXXXVIII five hundred eighty eight +DLXXXIX five hundred eighty nine +DXC five hundred ninety +DXCI five hundred ninety one +DXCII five hundred ninety two +DXCIII five hundred ninety three +DXCIV five hundred ninety four +DXCV five hundred ninety five +DXCVI five hundred ninety six +DXCVII five hundred ninety seven +DXCVIII five hundred ninety eight +DXCIX five hundred ninety nine +DC six hundred +DCI six hundred one +DCII six hundred two +DCIII six hundred three +DCIV six hundred four +DCV six hundred five +DCVI six hundred six +DCVII six hundred seven +DCVIII six hundred eight +DCIX six hundred nine +DCX six hundred ten +DCXI six hundred eleven +DCXII six hundred twelve +DCXIII six hundred thirteen +DCXIV six hundred fourteen +DCXV six hundred fifteen +DCXVI six hundred sixteen +DCXVII six hundred seventeen +DCXVIII six hundred eighteen +DCXIX six hundred nineteen +DCXX six hundred twenty +DCXXI six hundred twenty one +DCXXII six hundred twenty two +DCXXIII six hundred twenty three +DCXXIV six hundred twenty four +DCXXV six hundred twenty five +DCXXVI six hundred twenty six +DCXXVII six hundred twenty seven +DCXXVIII six hundred twenty eight +DCXXIX six hundred twenty nine +DCXXX six hundred thirty +DCXXXI six hundred thirty one +DCXXXII six hundred thirty two +DCXXXIII six hundred thirty three +DCXXXIV six hundred thirty four +DCXXXV six hundred thirty five +DCXXXVI six hundred thirty six +DCXXXVII six hundred thirty seven +DCXXXVIII six hundred thirty eight +DCXXXIX six hundred thirty nine +DCXL six hundred forty +DCXLI six hundred forty one +DCXLII six hundred forty two +DCXLIII six hundred forty three +DCXLIV six hundred forty four +DCXLV six hundred forty five +DCXLVI six hundred forty six +DCXLVII six hundred forty seven +DCXLVIII six hundred forty eight +DCXLIX six hundred forty nine +DCL six hundred fifty +DCLI six hundred fifty one +DCLII six hundred fifty two +DCLIII six hundred fifty three +DCLIV six hundred fifty four +DCLV six hundred fifty five +DCLVI six hundred fifty six +DCLVII six hundred fifty seven +DCLVIII six hundred fifty eight +DCLIX six hundred fifty nine +DCLX six hundred sixty +DCLXI six hundred sixty one +DCLXII six hundred sixty two +DCLXIII six hundred sixty three +DCLXIV six hundred sixty four +DCLXV six hundred sixty five +DCLXVI six hundred sixty six +DCLXVII six hundred sixty seven +DCLXVIII six hundred sixty eight +DCLXIX six hundred sixty nine +DCLXX six hundred seventy +DCLXXI six hundred seventy one +DCLXXII six hundred seventy two +DCLXXIII six hundred seventy three +DCLXXIV six hundred seventy four +DCLXXV six hundred seventy five +DCLXXVI six hundred seventy six +DCLXXVII six hundred seventy seven +DCLXXVIII six hundred seventy eight +DCLXXIX six hundred seventy nine +DCLXXX six hundred eighty +DCLXXXI six hundred eighty one +DCLXXXII six hundred eighty two +DCLXXXIII six hundred eighty three +DCLXXXIV six hundred eighty four +DCLXXXV six hundred eighty five +DCLXXXVI six hundred eighty six +DCLXXXVII six hundred eighty seven +DCLXXXVIII six hundred eighty eight +DCLXXXIX six hundred eighty nine +DCXC six hundred ninety +DCXCI six hundred ninety one +DCXCII six hundred ninety two +DCXCIII six hundred ninety three +DCXCIV six hundred ninety four +DCXCV six hundred ninety five +DCXCVI six hundred ninety six +DCXCVII six hundred ninety seven +DCXCVIII six hundred ninety eight +DCXCIX six hundred ninety nine +DCC seven hundred +DCCI seven hundred one +DCCII seven hundred two +DCCIII seven hundred three +DCCIV seven hundred four +DCCV seven hundred five +DCCVI seven hundred six +DCCVII seven hundred seven +DCCVIII seven hundred eight +DCCIX seven hundred nine +DCCX seven hundred ten +DCCXI seven hundred eleven +DCCXII seven hundred twelve +DCCXIII seven hundred thirteen +DCCXIV seven hundred fourteen +DCCXV seven hundred fifteen +DCCXVI seven hundred sixteen +DCCXVII seven hundred seventeen +DCCXVIII seven hundred eighteen +DCCXIX seven hundred nineteen +DCCXX seven hundred twenty +DCCXXI seven hundred twenty one +DCCXXII seven hundred twenty two +DCCXXIII seven hundred twenty three +DCCXXIV seven hundred twenty four +DCCXXV seven hundred twenty five +DCCXXVI seven hundred twenty six +DCCXXVII seven hundred twenty seven +DCCXXVIII seven hundred twenty eight +DCCXXIX seven hundred twenty nine +DCCXXX seven hundred thirty +DCCXXXI seven hundred thirty one +DCCXXXII seven hundred thirty two +DCCXXXIII seven hundred thirty three +DCCXXXIV seven hundred thirty four +DCCXXXV seven hundred thirty five +DCCXXXVI seven hundred thirty six +DCCXXXVII seven hundred thirty seven +DCCXXXVIII seven hundred thirty eight +DCCXXXIX seven hundred thirty nine +DCCXL seven hundred forty +DCCXLI seven hundred forty one +DCCXLII seven hundred forty two +DCCXLIII seven hundred forty three +DCCXLIV seven hundred forty four +DCCXLV seven hundred forty five +DCCXLVI seven hundred forty six +DCCXLVII seven hundred forty seven +DCCXLVIII seven hundred forty eight +DCCXLIX seven hundred forty nine +DCCL seven hundred fifty +DCCLI seven hundred fifty one +DCCLII seven hundred fifty two +DCCLIII seven hundred fifty three +DCCLIV seven hundred fifty four +DCCLV seven hundred fifty five +DCCLVI seven hundred fifty six +DCCLVII seven hundred fifty seven +DCCLVIII seven hundred fifty eight +DCCLIX seven hundred fifty nine +DCCLX seven hundred sixty +DCCLXI seven hundred sixty one +DCCLXII seven hundred sixty two +DCCLXIII seven hundred sixty three +DCCLXIV seven hundred sixty four +DCCLXV seven hundred sixty five +DCCLXVI seven hundred sixty six +DCCLXVII seven hundred sixty seven +DCCLXVIII seven hundred sixty eight +DCCLXIX seven hundred sixty nine +DCCLXX seven hundred seventy +DCCLXXI seven hundred seventy one +DCCLXXII seven hundred seventy two +DCCLXXIII seven hundred seventy three +DCCLXXIV seven hundred seventy four +DCCLXXV seven hundred seventy five +DCCLXXVI seven hundred seventy six +DCCLXXVII seven hundred seventy seven +DCCLXXVIII seven hundred seventy eight +DCCLXXIX seven hundred seventy nine +DCCLXXX seven hundred eighty +DCCLXXXI seven hundred eighty one +DCCLXXXII seven hundred eighty two +DCCLXXXIII seven hundred eighty three +DCCLXXXIV seven hundred eighty four +DCCLXXXV seven hundred eighty five +DCCLXXXVI seven hundred eighty six +DCCLXXXVII seven hundred eighty seven +DCCLXXXVIII seven hundred eighty eight +DCCLXXXIX seven hundred eighty nine +DCCXC seven hundred ninety +DCCXCI seven hundred ninety one +DCCXCII seven hundred ninety two +DCCXCIII seven hundred ninety three +DCCXCIV seven hundred ninety four +DCCXCV seven hundred ninety five +DCCXCVI seven hundred ninety six +DCCXCVII seven hundred ninety seven +DCCXCVIII seven hundred ninety eight +DCCXCIX seven hundred ninety nine +DCCC eight hundred +DCCCI eight hundred one +DCCCII eight hundred two +DCCCIII eight hundred three +DCCCIV eight hundred four +DCCCV eight hundred five +DCCCVI eight hundred six +DCCCVII eight hundred seven +DCCCVIII eight hundred eight +DCCCIX eight hundred nine +DCCCX eight hundred ten +DCCCXI eight hundred eleven +DCCCXII eight hundred twelve +DCCCXIII eight hundred thirteen +DCCCXIV eight hundred fourteen +DCCCXV eight hundred fifteen +DCCCXVI eight hundred sixteen +DCCCXVII eight hundred seventeen +DCCCXVIII eight hundred eighteen +DCCCXIX eight hundred nineteen +DCCCXX eight hundred twenty +DCCCXXI eight hundred twenty one +DCCCXXII eight hundred twenty two +DCCCXXIII eight hundred twenty three +DCCCXXIV eight hundred twenty four +DCCCXXV eight hundred twenty five +DCCCXXVI eight hundred twenty six +DCCCXXVII eight hundred twenty seven +DCCCXXVIII eight hundred twenty eight +DCCCXXIX eight hundred twenty nine +DCCCXXX eight hundred thirty +DCCCXXXI eight hundred thirty one +DCCCXXXII eight hundred thirty two +DCCCXXXIII eight hundred thirty three +DCCCXXXIV eight hundred thirty four +DCCCXXXV eight hundred thirty five +DCCCXXXVI eight hundred thirty six +DCCCXXXVII eight hundred thirty seven +DCCCXXXVIII eight hundred thirty eight +DCCCXXXIX eight hundred thirty nine +DCCCXL eight hundred forty +DCCCXLI eight hundred forty one +DCCCXLII eight hundred forty two +DCCCXLIII eight hundred forty three +DCCCXLIV eight hundred forty four +DCCCXLV eight hundred forty five +DCCCXLVI eight hundred forty six +DCCCXLVII eight hundred forty seven +DCCCXLVIII eight hundred forty eight +DCCCXLIX eight hundred forty nine +DCCCL eight hundred fifty +DCCCLI eight hundred fifty one +DCCCLII eight hundred fifty two +DCCCLIII eight hundred fifty three +DCCCLIV eight hundred fifty four +DCCCLV eight hundred fifty five +DCCCLVI eight hundred fifty six +DCCCLVII eight hundred fifty seven +DCCCLVIII eight hundred fifty eight +DCCCLIX eight hundred fifty nine +DCCCLX eight hundred sixty +DCCCLXI eight hundred sixty one +DCCCLXII eight hundred sixty two +DCCCLXIII eight hundred sixty three +DCCCLXIV eight hundred sixty four +DCCCLXV eight hundred sixty five +DCCCLXVI eight hundred sixty six +DCCCLXVII eight hundred sixty seven +DCCCLXVIII eight hundred sixty eight +DCCCLXIX eight hundred sixty nine +DCCCLXX eight hundred seventy +DCCCLXXI eight hundred seventy one +DCCCLXXII eight hundred seventy two +DCCCLXXIII eight hundred seventy three +DCCCLXXIV eight hundred seventy four +DCCCLXXV eight hundred seventy five +DCCCLXXVI eight hundred seventy six +DCCCLXXVII eight hundred seventy seven +DCCCLXXVIII eight hundred seventy eight +DCCCLXXIX eight hundred seventy nine +DCCCLXXX eight hundred eighty +DCCCLXXXI eight hundred eighty one +DCCCLXXXII eight hundred eighty two +DCCCLXXXIII eight hundred eighty three +DCCCLXXXIV eight hundred eighty four +DCCCLXXXV eight hundred eighty five +DCCCLXXXVI eight hundred eighty six +DCCCLXXXVII eight hundred eighty seven +DCCCLXXXVIII eight hundred eighty eight +DCCCLXXXIX eight hundred eighty nine +DCCCXC eight hundred ninety +DCCCXCI eight hundred ninety one +DCCCXCII eight hundred ninety two +DCCCXCIII eight hundred ninety three +DCCCXCIV eight hundred ninety four +DCCCXCV eight hundred ninety five +DCCCXCVI eight hundred ninety six +DCCCXCVII eight hundred ninety seven +DCCCXCVIII eight hundred ninety eight +DCCCXCIX eight hundred ninety nine +CM nine hundred +CMI nine hundred one +CMII nine hundred two +CMIII nine hundred three +CMIV nine hundred four +CMV nine hundred five +CMVI nine hundred six +CMVII nine hundred seven +CMVIII nine hundred eight +CMIX nine hundred nine +CMX nine hundred ten +CMXI nine hundred eleven +CMXII nine hundred twelve +CMXIII nine hundred thirteen +CMXIV nine hundred fourteen +CMXV nine hundred fifteen +CMXVI nine hundred sixteen +CMXVII nine hundred seventeen +CMXVIII nine hundred eighteen +CMXIX nine hundred nineteen +CMXX nine hundred twenty +CMXXI nine hundred twenty one +CMXXII nine hundred twenty two +CMXXIII nine hundred twenty three +CMXXIV nine hundred twenty four +CMXXV nine hundred twenty five +CMXXVI nine hundred twenty six +CMXXVII nine hundred twenty seven +CMXXVIII nine hundred twenty eight +CMXXIX nine hundred twenty nine +CMXXX nine hundred thirty +CMXXXI nine hundred thirty one +CMXXXII nine hundred thirty two +CMXXXIII nine hundred thirty three +CMXXXIV nine hundred thirty four +CMXXXV nine hundred thirty five +CMXXXVI nine hundred thirty six +CMXXXVII nine hundred thirty seven +CMXXXVIII nine hundred thirty eight +CMXXXIX nine hundred thirty nine +CMXL nine hundred forty +CMXLI nine hundred forty one +CMXLII nine hundred forty two +CMXLIII nine hundred forty three +CMXLIV nine hundred forty four +CMXLV nine hundred forty five +CMXLVI nine hundred forty six +CMXLVII nine hundred forty seven +CMXLVIII nine hundred forty eight +CMXLIX nine hundred forty nine +CML nine hundred fifty +CMLI nine hundred fifty one +CMLII nine hundred fifty two +CMLIII nine hundred fifty three +CMLIV nine hundred fifty four +CMLV nine hundred fifty five +CMLVI nine hundred fifty six +CMLVII nine hundred fifty seven +CMLVIII nine hundred fifty eight +CMLIX nine hundred fifty nine +CMLX nine hundred sixty +CMLXI nine hundred sixty one +CMLXII nine hundred sixty two +CMLXIII nine hundred sixty three +CMLXIV nine hundred sixty four +CMLXV nine hundred sixty five +CMLXVI nine hundred sixty six +CMLXVII nine hundred sixty seven +CMLXVIII nine hundred sixty eight +CMLXIX nine hundred sixty nine +CMLXX nine hundred seventy +CMLXXI nine hundred seventy one +CMLXXII nine hundred seventy two +CMLXXIII nine hundred seventy three +CMLXXIV nine hundred seventy four +CMLXXV nine hundred seventy five +CMLXXVI nine hundred seventy six +CMLXXVII nine hundred seventy seven +CMLXXVIII nine hundred seventy eight +CMLXXIX nine hundred seventy nine +CMLXXX nine hundred eighty +CMLXXXI nine hundred eighty one +CMLXXXII nine hundred eighty two +CMLXXXIII nine hundred eighty three +CMLXXXIV nine hundred eighty four +CMLXXXV nine hundred eighty five +CMLXXXVI nine hundred eighty six +CMLXXXVII nine hundred eighty seven +CMLXXXVIII nine hundred eighty eight +CMLXXXIX nine hundred eighty nine +CMXC nine hundred ninety +CMXCI nine hundred ninety one +CMXCII nine hundred ninety two +CMXCIII nine hundred ninety three +CMXCIV nine hundred ninety four +CMXCV nine hundred ninety five +CMXCVI nine hundred ninety six +CMXCVII nine hundred ninety seven +CMXCVIII nine hundred ninety eight +CMXCIX nine hundred ninety nine +M one thousand +MI one thousand one +MII one thousand two +MIII one thousand three +MIV one thousand four +MV one thousand five +MVI one thousand six +MVII one thousand seven +MVIII one thousand eight +MIX one thousand nine +MX one thousand ten +MXI one thousand eleven +MXII one thousand twelve +MXIII one thousand thirteen +MXIV one thousand fourteen +MXV one thousand fifteen +MXVI one thousand sixteen +MXVII one thousand seventeen +MXVIII one thousand eighteen +MXIX one thousand nineteen +MXX one thousand twenty +MXXI one thousand twenty one +MXXII one thousand twenty two +MXXIII one thousand twenty three +MXXIV one thousand twenty four +MXXV one thousand twenty five +MXXVI one thousand twenty six +MXXVII one thousand twenty seven +MXXVIII one thousand twenty eight +MXXIX one thousand twenty nine +MXXX one thousand thirty +MXXXI one thousand thirty one +MXXXII one thousand thirty two +MXXXIII one thousand thirty three +MXXXIV one thousand thirty four +MXXXV one thousand thirty five +MXXXVI one thousand thirty six +MXXXVII one thousand thirty seven +MXXXVIII one thousand thirty eight +MXXXIX one thousand thirty nine +MXL one thousand forty +MXLI one thousand forty one +MXLII one thousand forty two +MXLIII one thousand forty three +MXLIV one thousand forty four +MXLV one thousand forty five +MXLVI one thousand forty six +MXLVII one thousand forty seven +MXLVIII one thousand forty eight +MXLIX one thousand forty nine +ML one thousand fifty +MLI one thousand fifty one +MLII one thousand fifty two +MLIII one thousand fifty three +MLIV one thousand fifty four +MLV one thousand fifty five +MLVI one thousand fifty six +MLVII one thousand fifty seven +MLVIII one thousand fifty eight +MLIX one thousand fifty nine +MLX one thousand sixty +MLXI one thousand sixty one +MLXII one thousand sixty two +MLXIII one thousand sixty three +MLXIV one thousand sixty four +MLXV one thousand sixty five +MLXVI one thousand sixty six +MLXVII one thousand sixty seven +MLXVIII one thousand sixty eight +MLXIX one thousand sixty nine +MLXX one thousand seventy +MLXXI one thousand seventy one +MLXXII one thousand seventy two +MLXXIII one thousand seventy three +MLXXIV one thousand seventy four +MLXXV one thousand seventy five +MLXXVI one thousand seventy six +MLXXVII one thousand seventy seven +MLXXVIII one thousand seventy eight +MLXXIX one thousand seventy nine +MLXXX one thousand eighty +MLXXXI one thousand eighty one +MLXXXII one thousand eighty two +MLXXXIII one thousand eighty three +MLXXXIV one thousand eighty four +MLXXXV one thousand eighty five +MLXXXVI one thousand eighty six +MLXXXVII one thousand eighty seven +MLXXXVIII one thousand eighty eight +MLXXXIX one thousand eighty nine +MXC one thousand ninety +MXCI one thousand ninety one +MXCII one thousand ninety two +MXCIII one thousand ninety three +MXCIV one thousand ninety four +MXCV one thousand ninety five +MXCVI one thousand ninety six +MXCVII one thousand ninety seven +MXCVIII one thousand ninety eight +MXCIX one thousand ninety nine +MC one thousand, one hundred +MCI one thousand, one hundred one +MCII one thousand, one hundred two +MCIII one thousand, one hundred three +MCIV one thousand, one hundred four +MCV one thousand, one hundred five +MCVI one thousand, one hundred six +MCVII one thousand, one hundred seven +MCVIII one thousand, one hundred eight +MCIX one thousand, one hundred nine +MCX one thousand, one hundred ten +MCXI one thousand, one hundred eleven +MCXII one thousand, one hundred twelve +MCXIII one thousand, one hundred thirteen +MCXIV one thousand, one hundred fourteen +MCXV one thousand, one hundred fifteen +MCXVI one thousand, one hundred sixteen +MCXVII one thousand, one hundred seventeen +MCXVIII one thousand, one hundred eighteen +MCXIX one thousand, one hundred nineteen +MCXX one thousand, one hundred twenty +MCXXI one thousand, one hundred twenty one +MCXXII one thousand, one hundred twenty two +MCXXIII one thousand, one hundred twenty three +MCXXIV one thousand, one hundred twenty four +MCXXV one thousand, one hundred twenty five +MCXXVI one thousand, one hundred twenty six +MCXXVII one thousand, one hundred twenty seven +MCXXVIII one thousand, one hundred twenty eight +MCXXIX one thousand, one hundred twenty nine +MCXXX one thousand, one hundred thirty +MCXXXI one thousand, one hundred thirty one +MCXXXII one thousand, one hundred thirty two +MCXXXIII one thousand, one hundred thirty three +MCXXXIV one thousand, one hundred thirty four +MCXXXV one thousand, one hundred thirty five +MCXXXVI one thousand, one hundred thirty six +MCXXXVII one thousand, one hundred thirty seven +MCXXXVIII one thousand, one hundred thirty eight +MCXXXIX one thousand, one hundred thirty nine +MCXL one thousand, one hundred forty +MCXLI one thousand, one hundred forty one +MCXLII one thousand, one hundred forty two +MCXLIII one thousand, one hundred forty three +MCXLIV one thousand, one hundred forty four +MCXLV one thousand, one hundred forty five +MCXLVI one thousand, one hundred forty six +MCXLVII one thousand, one hundred forty seven +MCXLVIII one thousand, one hundred forty eight +MCXLIX one thousand, one hundred forty nine +MCL one thousand, one hundred fifty +MCLI one thousand, one hundred fifty one +MCLII one thousand, one hundred fifty two +MCLIII one thousand, one hundred fifty three +MCLIV one thousand, one hundred fifty four +MCLV one thousand, one hundred fifty five +MCLVI one thousand, one hundred fifty six +MCLVII one thousand, one hundred fifty seven +MCLVIII one thousand, one hundred fifty eight +MCLIX one thousand, one hundred fifty nine +MCLX one thousand, one hundred sixty +MCLXI one thousand, one hundred sixty one +MCLXII one thousand, one hundred sixty two +MCLXIII one thousand, one hundred sixty three +MCLXIV one thousand, one hundred sixty four +MCLXV one thousand, one hundred sixty five +MCLXVI one thousand, one hundred sixty six +MCLXVII one thousand, one hundred sixty seven +MCLXVIII one thousand, one hundred sixty eight +MCLXIX one thousand, one hundred sixty nine +MCLXX one thousand, one hundred seventy +MCLXXI one thousand, one hundred seventy one +MCLXXII one thousand, one hundred seventy two +MCLXXIII one thousand, one hundred seventy three +MCLXXIV one thousand, one hundred seventy four +MCLXXV one thousand, one hundred seventy five +MCLXXVI one thousand, one hundred seventy six +MCLXXVII one thousand, one hundred seventy seven +MCLXXVIII one thousand, one hundred seventy eight +MCLXXIX one thousand, one hundred seventy nine +MCLXXX one thousand, one hundred eighty +MCLXXXI one thousand, one hundred eighty one +MCLXXXII one thousand, one hundred eighty two +MCLXXXIII one thousand, one hundred eighty three +MCLXXXIV one thousand, one hundred eighty four +MCLXXXV one thousand, one hundred eighty five +MCLXXXVI one thousand, one hundred eighty six +MCLXXXVII one thousand, one hundred eighty seven +MCLXXXVIII one thousand, one hundred eighty eight +MCLXXXIX one thousand, one hundred eighty nine +MCXC one thousand, one hundred ninety +MCXCI one thousand, one hundred ninety one +MCXCII one thousand, one hundred ninety two +MCXCIII one thousand, one hundred ninety three +MCXCIV one thousand, one hundred ninety four +MCXCV one thousand, one hundred ninety five +MCXCVI one thousand, one hundred ninety six +MCXCVII one thousand, one hundred ninety seven +MCXCVIII one thousand, one hundred ninety eight +MCXCIX one thousand, one hundred ninety nine +MCC one thousand, two hundred +MCCI one thousand, two hundred one +MCCII one thousand, two hundred two +MCCIII one thousand, two hundred three +MCCIV one thousand, two hundred four +MCCV one thousand, two hundred five +MCCVI one thousand, two hundred six +MCCVII one thousand, two hundred seven +MCCVIII one thousand, two hundred eight +MCCIX one thousand, two hundred nine +MCCX one thousand, two hundred ten +MCCXI one thousand, two hundred eleven +MCCXII one thousand, two hundred twelve +MCCXIII one thousand, two hundred thirteen +MCCXIV one thousand, two hundred fourteen +MCCXV one thousand, two hundred fifteen +MCCXVI one thousand, two hundred sixteen +MCCXVII one thousand, two hundred seventeen +MCCXVIII one thousand, two hundred eighteen +MCCXIX one thousand, two hundred nineteen +MCCXX one thousand, two hundred twenty +MCCXXI one thousand, two hundred twenty one +MCCXXII one thousand, two hundred twenty two +MCCXXIII one thousand, two hundred twenty three +MCCXXIV one thousand, two hundred twenty four +MCCXXV one thousand, two hundred twenty five +MCCXXVI one thousand, two hundred twenty six +MCCXXVII one thousand, two hundred twenty seven +MCCXXVIII one thousand, two hundred twenty eight +MCCXXIX one thousand, two hundred twenty nine +MCCXXX one thousand, two hundred thirty +MCCXXXI one thousand, two hundred thirty one +MCCXXXII one thousand, two hundred thirty two +MCCXXXIII one thousand, two hundred thirty three +MCCXXXIV one thousand, two hundred thirty four +MCCXXXV one thousand, two hundred thirty five +MCCXXXVI one thousand, two hundred thirty six +MCCXXXVII one thousand, two hundred thirty seven +MCCXXXVIII one thousand, two hundred thirty eight +MCCXXXIX one thousand, two hundred thirty nine +MCCXL one thousand, two hundred forty +MCCXLI one thousand, two hundred forty one +MCCXLII one thousand, two hundred forty two +MCCXLIII one thousand, two hundred forty three +MCCXLIV one thousand, two hundred forty four +MCCXLV one thousand, two hundred forty five +MCCXLVI one thousand, two hundred forty six +MCCXLVII one thousand, two hundred forty seven +MCCXLVIII one thousand, two hundred forty eight +MCCXLIX one thousand, two hundred forty nine +MCCL one thousand, two hundred fifty +MCCLI one thousand, two hundred fifty one +MCCLII one thousand, two hundred fifty two +MCCLIII one thousand, two hundred fifty three +MCCLIV one thousand, two hundred fifty four +MCCLV one thousand, two hundred fifty five +MCCLVI one thousand, two hundred fifty six +MCCLVII one thousand, two hundred fifty seven +MCCLVIII one thousand, two hundred fifty eight +MCCLIX one thousand, two hundred fifty nine +MCCLX one thousand, two hundred sixty +MCCLXI one thousand, two hundred sixty one +MCCLXII one thousand, two hundred sixty two +MCCLXIII one thousand, two hundred sixty three +MCCLXIV one thousand, two hundred sixty four +MCCLXV one thousand, two hundred sixty five +MCCLXVI one thousand, two hundred sixty six +MCCLXVII one thousand, two hundred sixty seven +MCCLXVIII one thousand, two hundred sixty eight +MCCLXIX one thousand, two hundred sixty nine +MCCLXX one thousand, two hundred seventy +MCCLXXI one thousand, two hundred seventy one +MCCLXXII one thousand, two hundred seventy two +MCCLXXIII one thousand, two hundred seventy three +MCCLXXIV one thousand, two hundred seventy four +MCCLXXV one thousand, two hundred seventy five +MCCLXXVI one thousand, two hundred seventy six +MCCLXXVII one thousand, two hundred seventy seven +MCCLXXVIII one thousand, two hundred seventy eight +MCCLXXIX one thousand, two hundred seventy nine +MCCLXXX one thousand, two hundred eighty +MCCLXXXI one thousand, two hundred eighty one +MCCLXXXII one thousand, two hundred eighty two +MCCLXXXIII one thousand, two hundred eighty three +MCCLXXXIV one thousand, two hundred eighty four +MCCLXXXV one thousand, two hundred eighty five +MCCLXXXVI one thousand, two hundred eighty six +MCCLXXXVII one thousand, two hundred eighty seven +MCCLXXXVIII one thousand, two hundred eighty eight +MCCLXXXIX one thousand, two hundred eighty nine +MCCXC one thousand, two hundred ninety +MCCXCI one thousand, two hundred ninety one +MCCXCII one thousand, two hundred ninety two +MCCXCIII one thousand, two hundred ninety three +MCCXCIV one thousand, two hundred ninety four +MCCXCV one thousand, two hundred ninety five +MCCXCVI one thousand, two hundred ninety six +MCCXCVII one thousand, two hundred ninety seven +MCCXCVIII one thousand, two hundred ninety eight +MCCXCIX one thousand, two hundred ninety nine +MCCC one thousand, three hundred +MCCCI one thousand, three hundred one +MCCCII one thousand, three hundred two +MCCCIII one thousand, three hundred three +MCCCIV one thousand, three hundred four +MCCCV one thousand, three hundred five +MCCCVI one thousand, three hundred six +MCCCVII one thousand, three hundred seven +MCCCVIII one thousand, three hundred eight +MCCCIX one thousand, three hundred nine +MCCCX one thousand, three hundred ten +MCCCXI one thousand, three hundred eleven +MCCCXII one thousand, three hundred twelve +MCCCXIII one thousand, three hundred thirteen +MCCCXIV one thousand, three hundred fourteen +MCCCXV one thousand, three hundred fifteen +MCCCXVI one thousand, three hundred sixteen +MCCCXVII one thousand, three hundred seventeen +MCCCXVIII one thousand, three hundred eighteen +MCCCXIX one thousand, three hundred nineteen +MCCCXX one thousand, three hundred twenty +MCCCXXI one thousand, three hundred twenty one +MCCCXXII one thousand, three hundred twenty two +MCCCXXIII one thousand, three hundred twenty three +MCCCXXIV one thousand, three hundred twenty four +MCCCXXV one thousand, three hundred twenty five +MCCCXXVI one thousand, three hundred twenty six +MCCCXXVII one thousand, three hundred twenty seven +MCCCXXVIII one thousand, three hundred twenty eight +MCCCXXIX one thousand, three hundred twenty nine +MCCCXXX one thousand, three hundred thirty +MCCCXXXI one thousand, three hundred thirty one +MCCCXXXII one thousand, three hundred thirty two +MCCCXXXIII one thousand, three hundred thirty three +MCCCXXXIV one thousand, three hundred thirty four +MCCCXXXV one thousand, three hundred thirty five +MCCCXXXVI one thousand, three hundred thirty six +MCCCXXXVII one thousand, three hundred thirty seven +MCCCXXXVIII one thousand, three hundred thirty eight +MCCCXXXIX one thousand, three hundred thirty nine +MCCCXL one thousand, three hundred forty +MCCCXLI one thousand, three hundred forty one +MCCCXLII one thousand, three hundred forty two +MCCCXLIII one thousand, three hundred forty three +MCCCXLIV one thousand, three hundred forty four +MCCCXLV one thousand, three hundred forty five +MCCCXLVI one thousand, three hundred forty six +MCCCXLVII one thousand, three hundred forty seven +MCCCXLVIII one thousand, three hundred forty eight +MCCCXLIX one thousand, three hundred forty nine +MCCCL one thousand, three hundred fifty +MCCCLI one thousand, three hundred fifty one +MCCCLII one thousand, three hundred fifty two +MCCCLIII one thousand, three hundred fifty three +MCCCLIV one thousand, three hundred fifty four +MCCCLV one thousand, three hundred fifty five +MCCCLVI one thousand, three hundred fifty six +MCCCLVII one thousand, three hundred fifty seven +MCCCLVIII one thousand, three hundred fifty eight +MCCCLIX one thousand, three hundred fifty nine +MCCCLX one thousand, three hundred sixty +MCCCLXI one thousand, three hundred sixty one +MCCCLXII one thousand, three hundred sixty two +MCCCLXIII one thousand, three hundred sixty three +MCCCLXIV one thousand, three hundred sixty four +MCCCLXV one thousand, three hundred sixty five +MCCCLXVI one thousand, three hundred sixty six +MCCCLXVII one thousand, three hundred sixty seven +MCCCLXVIII one thousand, three hundred sixty eight +MCCCLXIX one thousand, three hundred sixty nine +MCCCLXX one thousand, three hundred seventy +MCCCLXXI one thousand, three hundred seventy one +MCCCLXXII one thousand, three hundred seventy two +MCCCLXXIII one thousand, three hundred seventy three +MCCCLXXIV one thousand, three hundred seventy four +MCCCLXXV one thousand, three hundred seventy five +MCCCLXXVI one thousand, three hundred seventy six +MCCCLXXVII one thousand, three hundred seventy seven +MCCCLXXVIII one thousand, three hundred seventy eight +MCCCLXXIX one thousand, three hundred seventy nine +MCCCLXXX one thousand, three hundred eighty +MCCCLXXXI one thousand, three hundred eighty one +MCCCLXXXII one thousand, three hundred eighty two +MCCCLXXXIII one thousand, three hundred eighty three +MCCCLXXXIV one thousand, three hundred eighty four +MCCCLXXXV one thousand, three hundred eighty five +MCCCLXXXVI one thousand, three hundred eighty six +MCCCLXXXVII one thousand, three hundred eighty seven +MCCCLXXXVIII one thousand, three hundred eighty eight +MCCCLXXXIX one thousand, three hundred eighty nine +MCCCXC one thousand, three hundred ninety +MCCCXCI one thousand, three hundred ninety one +MCCCXCII one thousand, three hundred ninety two +MCCCXCIII one thousand, three hundred ninety three +MCCCXCIV one thousand, three hundred ninety four +MCCCXCV one thousand, three hundred ninety five +MCCCXCVI one thousand, three hundred ninety six +MCCCXCVII one thousand, three hundred ninety seven +MCCCXCVIII one thousand, three hundred ninety eight +MCCCXCIX one thousand, three hundred ninety nine +MCD one thousand, four hundred +MCDI one thousand, four hundred one +MCDII one thousand, four hundred two +MCDIII one thousand, four hundred three +MCDIV one thousand, four hundred four +MCDV one thousand, four hundred five +MCDVI one thousand, four hundred six +MCDVII one thousand, four hundred seven +MCDVIII one thousand, four hundred eight +MCDIX one thousand, four hundred nine +MCDX one thousand, four hundred ten +MCDXI one thousand, four hundred eleven +MCDXII one thousand, four hundred twelve +MCDXIII one thousand, four hundred thirteen +MCDXIV one thousand, four hundred fourteen +MCDXV one thousand, four hundred fifteen +MCDXVI one thousand, four hundred sixteen +MCDXVII one thousand, four hundred seventeen +MCDXVIII one thousand, four hundred eighteen +MCDXIX one thousand, four hundred nineteen +MCDXX one thousand, four hundred twenty +MCDXXI one thousand, four hundred twenty one +MCDXXII one thousand, four hundred twenty two +MCDXXIII one thousand, four hundred twenty three +MCDXXIV one thousand, four hundred twenty four +MCDXXV one thousand, four hundred twenty five +MCDXXVI one thousand, four hundred twenty six +MCDXXVII one thousand, four hundred twenty seven +MCDXXVIII one thousand, four hundred twenty eight +MCDXXIX one thousand, four hundred twenty nine +MCDXXX one thousand, four hundred thirty +MCDXXXI one thousand, four hundred thirty one +MCDXXXII one thousand, four hundred thirty two +MCDXXXIII one thousand, four hundred thirty three +MCDXXXIV one thousand, four hundred thirty four +MCDXXXV one thousand, four hundred thirty five +MCDXXXVI one thousand, four hundred thirty six +MCDXXXVII one thousand, four hundred thirty seven +MCDXXXVIII one thousand, four hundred thirty eight +MCDXXXIX one thousand, four hundred thirty nine +MCDXL one thousand, four hundred forty +MCDXLI one thousand, four hundred forty one +MCDXLII one thousand, four hundred forty two +MCDXLIII one thousand, four hundred forty three +MCDXLIV one thousand, four hundred forty four +MCDXLV one thousand, four hundred forty five +MCDXLVI one thousand, four hundred forty six +MCDXLVII one thousand, four hundred forty seven +MCDXLVIII one thousand, four hundred forty eight +MCDXLIX one thousand, four hundred forty nine +MCDL one thousand, four hundred fifty +MCDLI one thousand, four hundred fifty one +MCDLII one thousand, four hundred fifty two +MCDLIII one thousand, four hundred fifty three +MCDLIV one thousand, four hundred fifty four +MCDLV one thousand, four hundred fifty five +MCDLVI one thousand, four hundred fifty six +MCDLVII one thousand, four hundred fifty seven +MCDLVIII one thousand, four hundred fifty eight +MCDLIX one thousand, four hundred fifty nine +MCDLX one thousand, four hundred sixty +MCDLXI one thousand, four hundred sixty one +MCDLXII one thousand, four hundred sixty two +MCDLXIII one thousand, four hundred sixty three +MCDLXIV one thousand, four hundred sixty four +MCDLXV one thousand, four hundred sixty five +MCDLXVI one thousand, four hundred sixty six +MCDLXVII one thousand, four hundred sixty seven +MCDLXVIII one thousand, four hundred sixty eight +MCDLXIX one thousand, four hundred sixty nine +MCDLXX one thousand, four hundred seventy +MCDLXXI one thousand, four hundred seventy one +MCDLXXII one thousand, four hundred seventy two +MCDLXXIII one thousand, four hundred seventy three +MCDLXXIV one thousand, four hundred seventy four +MCDLXXV one thousand, four hundred seventy five +MCDLXXVI one thousand, four hundred seventy six +MCDLXXVII one thousand, four hundred seventy seven +MCDLXXVIII one thousand, four hundred seventy eight +MCDLXXIX one thousand, four hundred seventy nine +MCDLXXX one thousand, four hundred eighty +MCDLXXXI one thousand, four hundred eighty one +MCDLXXXII one thousand, four hundred eighty two +MCDLXXXIII one thousand, four hundred eighty three +MCDLXXXIV one thousand, four hundred eighty four +MCDLXXXV one thousand, four hundred eighty five +MCDLXXXVI one thousand, four hundred eighty six +MCDLXXXVII one thousand, four hundred eighty seven +MCDLXXXVIII one thousand, four hundred eighty eight +MCDLXXXIX one thousand, four hundred eighty nine +MCDXC one thousand, four hundred ninety +MCDXCI one thousand, four hundred ninety one +MCDXCII one thousand, four hundred ninety two +MCDXCIII one thousand, four hundred ninety three +MCDXCIV one thousand, four hundred ninety four +MCDXCV one thousand, four hundred ninety five +MCDXCVI one thousand, four hundred ninety six +MCDXCVII one thousand, four hundred ninety seven +MCDXCVIII one thousand, four hundred ninety eight +MCDXCIX one thousand, four hundred ninety nine +MD one thousand, five hundred +MDI one thousand, five hundred one +MDII one thousand, five hundred two +MDIII one thousand, five hundred three +MDIV one thousand, five hundred four +MDV one thousand, five hundred five +MDVI one thousand, five hundred six +MDVII one thousand, five hundred seven +MDVIII one thousand, five hundred eight +MDIX one thousand, five hundred nine +MDX one thousand, five hundred ten +MDXI one thousand, five hundred eleven +MDXII one thousand, five hundred twelve +MDXIII one thousand, five hundred thirteen +MDXIV one thousand, five hundred fourteen +MDXV one thousand, five hundred fifteen +MDXVI one thousand, five hundred sixteen +MDXVII one thousand, five hundred seventeen +MDXVIII one thousand, five hundred eighteen +MDXIX one thousand, five hundred nineteen +MDXX one thousand, five hundred twenty +MDXXI one thousand, five hundred twenty one +MDXXII one thousand, five hundred twenty two +MDXXIII one thousand, five hundred twenty three +MDXXIV one thousand, five hundred twenty four +MDXXV one thousand, five hundred twenty five +MDXXVI one thousand, five hundred twenty six +MDXXVII one thousand, five hundred twenty seven +MDXXVIII one thousand, five hundred twenty eight +MDXXIX one thousand, five hundred twenty nine +MDXXX one thousand, five hundred thirty +MDXXXI one thousand, five hundred thirty one +MDXXXII one thousand, five hundred thirty two +MDXXXIII one thousand, five hundred thirty three +MDXXXIV one thousand, five hundred thirty four +MDXXXV one thousand, five hundred thirty five +MDXXXVI one thousand, five hundred thirty six +MDXXXVII one thousand, five hundred thirty seven +MDXXXVIII one thousand, five hundred thirty eight +MDXXXIX one thousand, five hundred thirty nine +MDXL one thousand, five hundred forty +MDXLI one thousand, five hundred forty one +MDXLII one thousand, five hundred forty two +MDXLIII one thousand, five hundred forty three +MDXLIV one thousand, five hundred forty four +MDXLV one thousand, five hundred forty five +MDXLVI one thousand, five hundred forty six +MDXLVII one thousand, five hundred forty seven +MDXLVIII one thousand, five hundred forty eight +MDXLIX one thousand, five hundred forty nine +MDL one thousand, five hundred fifty +MDLI one thousand, five hundred fifty one +MDLII one thousand, five hundred fifty two +MDLIII one thousand, five hundred fifty three +MDLIV one thousand, five hundred fifty four +MDLV one thousand, five hundred fifty five +MDLVI one thousand, five hundred fifty six +MDLVII one thousand, five hundred fifty seven +MDLVIII one thousand, five hundred fifty eight +MDLIX one thousand, five hundred fifty nine +MDLX one thousand, five hundred sixty +MDLXI one thousand, five hundred sixty one +MDLXII one thousand, five hundred sixty two +MDLXIII one thousand, five hundred sixty three +MDLXIV one thousand, five hundred sixty four +MDLXV one thousand, five hundred sixty five +MDLXVI one thousand, five hundred sixty six +MDLXVII one thousand, five hundred sixty seven +MDLXVIII one thousand, five hundred sixty eight +MDLXIX one thousand, five hundred sixty nine +MDLXX one thousand, five hundred seventy +MDLXXI one thousand, five hundred seventy one +MDLXXII one thousand, five hundred seventy two +MDLXXIII one thousand, five hundred seventy three +MDLXXIV one thousand, five hundred seventy four +MDLXXV one thousand, five hundred seventy five +MDLXXVI one thousand, five hundred seventy six +MDLXXVII one thousand, five hundred seventy seven +MDLXXVIII one thousand, five hundred seventy eight +MDLXXIX one thousand, five hundred seventy nine +MDLXXX one thousand, five hundred eighty +MDLXXXI one thousand, five hundred eighty one +MDLXXXII one thousand, five hundred eighty two +MDLXXXIII one thousand, five hundred eighty three +MDLXXXIV one thousand, five hundred eighty four +MDLXXXV one thousand, five hundred eighty five +MDLXXXVI one thousand, five hundred eighty six +MDLXXXVII one thousand, five hundred eighty seven +MDLXXXVIII one thousand, five hundred eighty eight +MDLXXXIX one thousand, five hundred eighty nine +MDXC one thousand, five hundred ninety +MDXCI one thousand, five hundred ninety one +MDXCII one thousand, five hundred ninety two +MDXCIII one thousand, five hundred ninety three +MDXCIV one thousand, five hundred ninety four +MDXCV one thousand, five hundred ninety five +MDXCVI one thousand, five hundred ninety six +MDXCVII one thousand, five hundred ninety seven +MDXCVIII one thousand, five hundred ninety eight +MDXCIX one thousand, five hundred ninety nine +MDC one thousand, six hundred +MDCI one thousand, six hundred one +MDCII one thousand, six hundred two +MDCIII one thousand, six hundred three +MDCIV one thousand, six hundred four +MDCV one thousand, six hundred five +MDCVI one thousand, six hundred six +MDCVII one thousand, six hundred seven +MDCVIII one thousand, six hundred eight +MDCIX one thousand, six hundred nine +MDCX one thousand, six hundred ten +MDCXI one thousand, six hundred eleven +MDCXII one thousand, six hundred twelve +MDCXIII one thousand, six hundred thirteen +MDCXIV one thousand, six hundred fourteen +MDCXV one thousand, six hundred fifteen +MDCXVI one thousand, six hundred sixteen +MDCXVII one thousand, six hundred seventeen +MDCXVIII one thousand, six hundred eighteen +MDCXIX one thousand, six hundred nineteen +MDCXX one thousand, six hundred twenty +MDCXXI one thousand, six hundred twenty one +MDCXXII one thousand, six hundred twenty two +MDCXXIII one thousand, six hundred twenty three +MDCXXIV one thousand, six hundred twenty four +MDCXXV one thousand, six hundred twenty five +MDCXXVI one thousand, six hundred twenty six +MDCXXVII one thousand, six hundred twenty seven +MDCXXVIII one thousand, six hundred twenty eight +MDCXXIX one thousand, six hundred twenty nine +MDCXXX one thousand, six hundred thirty +MDCXXXI one thousand, six hundred thirty one +MDCXXXII one thousand, six hundred thirty two +MDCXXXIII one thousand, six hundred thirty three +MDCXXXIV one thousand, six hundred thirty four +MDCXXXV one thousand, six hundred thirty five +MDCXXXVI one thousand, six hundred thirty six +MDCXXXVII one thousand, six hundred thirty seven +MDCXXXVIII one thousand, six hundred thirty eight +MDCXXXIX one thousand, six hundred thirty nine +MDCXL one thousand, six hundred forty +MDCXLI one thousand, six hundred forty one +MDCXLII one thousand, six hundred forty two +MDCXLIII one thousand, six hundred forty three +MDCXLIV one thousand, six hundred forty four +MDCXLV one thousand, six hundred forty five +MDCXLVI one thousand, six hundred forty six +MDCXLVII one thousand, six hundred forty seven +MDCXLVIII one thousand, six hundred forty eight +MDCXLIX one thousand, six hundred forty nine +MDCL one thousand, six hundred fifty +MDCLI one thousand, six hundred fifty one +MDCLII one thousand, six hundred fifty two +MDCLIII one thousand, six hundred fifty three +MDCLIV one thousand, six hundred fifty four +MDCLV one thousand, six hundred fifty five +MDCLVI one thousand, six hundred fifty six +MDCLVII one thousand, six hundred fifty seven +MDCLVIII one thousand, six hundred fifty eight +MDCLIX one thousand, six hundred fifty nine +MDCLX one thousand, six hundred sixty +MDCLXI one thousand, six hundred sixty one +MDCLXII one thousand, six hundred sixty two +MDCLXIII one thousand, six hundred sixty three +MDCLXIV one thousand, six hundred sixty four +MDCLXV one thousand, six hundred sixty five +MDCLXVI one thousand, six hundred sixty six +MDCLXVII one thousand, six hundred sixty seven +MDCLXVIII one thousand, six hundred sixty eight +MDCLXIX one thousand, six hundred sixty nine +MDCLXX one thousand, six hundred seventy +MDCLXXI one thousand, six hundred seventy one +MDCLXXII one thousand, six hundred seventy two +MDCLXXIII one thousand, six hundred seventy three +MDCLXXIV one thousand, six hundred seventy four +MDCLXXV one thousand, six hundred seventy five +MDCLXXVI one thousand, six hundred seventy six +MDCLXXVII one thousand, six hundred seventy seven +MDCLXXVIII one thousand, six hundred seventy eight +MDCLXXIX one thousand, six hundred seventy nine +MDCLXXX one thousand, six hundred eighty +MDCLXXXI one thousand, six hundred eighty one +MDCLXXXII one thousand, six hundred eighty two +MDCLXXXIII one thousand, six hundred eighty three +MDCLXXXIV one thousand, six hundred eighty four +MDCLXXXV one thousand, six hundred eighty five +MDCLXXXVI one thousand, six hundred eighty six +MDCLXXXVII one thousand, six hundred eighty seven +MDCLXXXVIII one thousand, six hundred eighty eight +MDCLXXXIX one thousand, six hundred eighty nine +MDCXC one thousand, six hundred ninety +MDCXCI one thousand, six hundred ninety one +MDCXCII one thousand, six hundred ninety two +MDCXCIII one thousand, six hundred ninety three +MDCXCIV one thousand, six hundred ninety four +MDCXCV one thousand, six hundred ninety five +MDCXCVI one thousand, six hundred ninety six +MDCXCVII one thousand, six hundred ninety seven +MDCXCVIII one thousand, six hundred ninety eight +MDCXCIX one thousand, six hundred ninety nine +MDCC one thousand, seven hundred +MDCCI one thousand, seven hundred one +MDCCII one thousand, seven hundred two +MDCCIII one thousand, seven hundred three +MDCCIV one thousand, seven hundred four +MDCCV one thousand, seven hundred five +MDCCVI one thousand, seven hundred six +MDCCVII one thousand, seven hundred seven +MDCCVIII one thousand, seven hundred eight +MDCCIX one thousand, seven hundred nine +MDCCX one thousand, seven hundred ten +MDCCXI one thousand, seven hundred eleven +MDCCXII one thousand, seven hundred twelve +MDCCXIII one thousand, seven hundred thirteen +MDCCXIV one thousand, seven hundred fourteen +MDCCXV one thousand, seven hundred fifteen +MDCCXVI one thousand, seven hundred sixteen +MDCCXVII one thousand, seven hundred seventeen +MDCCXVIII one thousand, seven hundred eighteen +MDCCXIX one thousand, seven hundred nineteen +MDCCXX one thousand, seven hundred twenty +MDCCXXI one thousand, seven hundred twenty one +MDCCXXII one thousand, seven hundred twenty two +MDCCXXIII one thousand, seven hundred twenty three +MDCCXXIV one thousand, seven hundred twenty four +MDCCXXV one thousand, seven hundred twenty five +MDCCXXVI one thousand, seven hundred twenty six +MDCCXXVII one thousand, seven hundred twenty seven +MDCCXXVIII one thousand, seven hundred twenty eight +MDCCXXIX one thousand, seven hundred twenty nine +MDCCXXX one thousand, seven hundred thirty +MDCCXXXI one thousand, seven hundred thirty one +MDCCXXXII one thousand, seven hundred thirty two +MDCCXXXIII one thousand, seven hundred thirty three +MDCCXXXIV one thousand, seven hundred thirty four +MDCCXXXV one thousand, seven hundred thirty five +MDCCXXXVI one thousand, seven hundred thirty six +MDCCXXXVII one thousand, seven hundred thirty seven +MDCCXXXVIII one thousand, seven hundred thirty eight +MDCCXXXIX one thousand, seven hundred thirty nine +MDCCXL one thousand, seven hundred forty +MDCCXLI one thousand, seven hundred forty one +MDCCXLII one thousand, seven hundred forty two +MDCCXLIII one thousand, seven hundred forty three +MDCCXLIV one thousand, seven hundred forty four +MDCCXLV one thousand, seven hundred forty five +MDCCXLVI one thousand, seven hundred forty six +MDCCXLVII one thousand, seven hundred forty seven +MDCCXLVIII one thousand, seven hundred forty eight +MDCCXLIX one thousand, seven hundred forty nine +MDCCL one thousand, seven hundred fifty +MDCCLI one thousand, seven hundred fifty one +MDCCLII one thousand, seven hundred fifty two +MDCCLIII one thousand, seven hundred fifty three +MDCCLIV one thousand, seven hundred fifty four +MDCCLV one thousand, seven hundred fifty five +MDCCLVI one thousand, seven hundred fifty six +MDCCLVII one thousand, seven hundred fifty seven +MDCCLVIII one thousand, seven hundred fifty eight +MDCCLIX one thousand, seven hundred fifty nine +MDCCLX one thousand, seven hundred sixty +MDCCLXI one thousand, seven hundred sixty one +MDCCLXII one thousand, seven hundred sixty two +MDCCLXIII one thousand, seven hundred sixty three +MDCCLXIV one thousand, seven hundred sixty four +MDCCLXV one thousand, seven hundred sixty five +MDCCLXVI one thousand, seven hundred sixty six +MDCCLXVII one thousand, seven hundred sixty seven +MDCCLXVIII one thousand, seven hundred sixty eight +MDCCLXIX one thousand, seven hundred sixty nine +MDCCLXX one thousand, seven hundred seventy +MDCCLXXI one thousand, seven hundred seventy one +MDCCLXXII one thousand, seven hundred seventy two +MDCCLXXIII one thousand, seven hundred seventy three +MDCCLXXIV one thousand, seven hundred seventy four +MDCCLXXV one thousand, seven hundred seventy five +MDCCLXXVI one thousand, seven hundred seventy six +MDCCLXXVII one thousand, seven hundred seventy seven +MDCCLXXVIII one thousand, seven hundred seventy eight +MDCCLXXIX one thousand, seven hundred seventy nine +MDCCLXXX one thousand, seven hundred eighty +MDCCLXXXI one thousand, seven hundred eighty one +MDCCLXXXII one thousand, seven hundred eighty two +MDCCLXXXIII one thousand, seven hundred eighty three +MDCCLXXXIV one thousand, seven hundred eighty four +MDCCLXXXV one thousand, seven hundred eighty five +MDCCLXXXVI one thousand, seven hundred eighty six +MDCCLXXXVII one thousand, seven hundred eighty seven +MDCCLXXXVIII one thousand, seven hundred eighty eight +MDCCLXXXIX one thousand, seven hundred eighty nine +MDCCXC one thousand, seven hundred ninety +MDCCXCI one thousand, seven hundred ninety one +MDCCXCII one thousand, seven hundred ninety two +MDCCXCIII one thousand, seven hundred ninety three +MDCCXCIV one thousand, seven hundred ninety four +MDCCXCV one thousand, seven hundred ninety five +MDCCXCVI one thousand, seven hundred ninety six +MDCCXCVII one thousand, seven hundred ninety seven +MDCCXCVIII one thousand, seven hundred ninety eight +MDCCXCIX one thousand, seven hundred ninety nine +MDCCC one thousand, eight hundred +MDCCCI one thousand, eight hundred one +MDCCCII one thousand, eight hundred two +MDCCCIII one thousand, eight hundred three +MDCCCIV one thousand, eight hundred four +MDCCCV one thousand, eight hundred five +MDCCCVI one thousand, eight hundred six +MDCCCVII one thousand, eight hundred seven +MDCCCVIII one thousand, eight hundred eight +MDCCCIX one thousand, eight hundred nine +MDCCCX one thousand, eight hundred ten +MDCCCXI one thousand, eight hundred eleven +MDCCCXII one thousand, eight hundred twelve +MDCCCXIII one thousand, eight hundred thirteen +MDCCCXIV one thousand, eight hundred fourteen +MDCCCXV one thousand, eight hundred fifteen +MDCCCXVI one thousand, eight hundred sixteen +MDCCCXVII one thousand, eight hundred seventeen +MDCCCXVIII one thousand, eight hundred eighteen +MDCCCXIX one thousand, eight hundred nineteen +MDCCCXX one thousand, eight hundred twenty +MDCCCXXI one thousand, eight hundred twenty one +MDCCCXXII one thousand, eight hundred twenty two +MDCCCXXIII one thousand, eight hundred twenty three +MDCCCXXIV one thousand, eight hundred twenty four +MDCCCXXV one thousand, eight hundred twenty five +MDCCCXXVI one thousand, eight hundred twenty six +MDCCCXXVII one thousand, eight hundred twenty seven +MDCCCXXVIII one thousand, eight hundred twenty eight +MDCCCXXIX one thousand, eight hundred twenty nine +MDCCCXXX one thousand, eight hundred thirty +MDCCCXXXI one thousand, eight hundred thirty one +MDCCCXXXII one thousand, eight hundred thirty two +MDCCCXXXIII one thousand, eight hundred thirty three +MDCCCXXXIV one thousand, eight hundred thirty four +MDCCCXXXV one thousand, eight hundred thirty five +MDCCCXXXVI one thousand, eight hundred thirty six +MDCCCXXXVII one thousand, eight hundred thirty seven +MDCCCXXXVIII one thousand, eight hundred thirty eight +MDCCCXXXIX one thousand, eight hundred thirty nine +MDCCCXL one thousand, eight hundred forty +MDCCCXLI one thousand, eight hundred forty one +MDCCCXLII one thousand, eight hundred forty two +MDCCCXLIII one thousand, eight hundred forty three +MDCCCXLIV one thousand, eight hundred forty four +MDCCCXLV one thousand, eight hundred forty five +MDCCCXLVI one thousand, eight hundred forty six +MDCCCXLVII one thousand, eight hundred forty seven +MDCCCXLVIII one thousand, eight hundred forty eight +MDCCCXLIX one thousand, eight hundred forty nine +MDCCCL one thousand, eight hundred fifty +MDCCCLI one thousand, eight hundred fifty one +MDCCCLII one thousand, eight hundred fifty two +MDCCCLIII one thousand, eight hundred fifty three +MDCCCLIV one thousand, eight hundred fifty four +MDCCCLV one thousand, eight hundred fifty five +MDCCCLVI one thousand, eight hundred fifty six +MDCCCLVII one thousand, eight hundred fifty seven +MDCCCLVIII one thousand, eight hundred fifty eight +MDCCCLIX one thousand, eight hundred fifty nine +MDCCCLX one thousand, eight hundred sixty +MDCCCLXI one thousand, eight hundred sixty one +MDCCCLXII one thousand, eight hundred sixty two +MDCCCLXIII one thousand, eight hundred sixty three +MDCCCLXIV one thousand, eight hundred sixty four +MDCCCLXV one thousand, eight hundred sixty five +MDCCCLXVI one thousand, eight hundred sixty six +MDCCCLXVII one thousand, eight hundred sixty seven +MDCCCLXVIII one thousand, eight hundred sixty eight +MDCCCLXIX one thousand, eight hundred sixty nine +MDCCCLXX one thousand, eight hundred seventy +MDCCCLXXI one thousand, eight hundred seventy one +MDCCCLXXII one thousand, eight hundred seventy two +MDCCCLXXIII one thousand, eight hundred seventy three +MDCCCLXXIV one thousand, eight hundred seventy four +MDCCCLXXV one thousand, eight hundred seventy five +MDCCCLXXVI one thousand, eight hundred seventy six +MDCCCLXXVII one thousand, eight hundred seventy seven +MDCCCLXXVIII one thousand, eight hundred seventy eight +MDCCCLXXIX one thousand, eight hundred seventy nine +MDCCCLXXX one thousand, eight hundred eighty +MDCCCLXXXI one thousand, eight hundred eighty one +MDCCCLXXXII one thousand, eight hundred eighty two +MDCCCLXXXIII one thousand, eight hundred eighty three +MDCCCLXXXIV one thousand, eight hundred eighty four +MDCCCLXXXV one thousand, eight hundred eighty five +MDCCCLXXXVI one thousand, eight hundred eighty six +MDCCCLXXXVII one thousand, eight hundred eighty seven +MDCCCLXXXVIII one thousand, eight hundred eighty eight +MDCCCLXXXIX one thousand, eight hundred eighty nine +MDCCCXC one thousand, eight hundred ninety +MDCCCXCI one thousand, eight hundred ninety one +MDCCCXCII one thousand, eight hundred ninety two +MDCCCXCIII one thousand, eight hundred ninety three +MDCCCXCIV one thousand, eight hundred ninety four +MDCCCXCV one thousand, eight hundred ninety five +MDCCCXCVI one thousand, eight hundred ninety six +MDCCCXCVII one thousand, eight hundred ninety seven +MDCCCXCVIII one thousand, eight hundred ninety eight +MDCCCXCIX one thousand, eight hundred ninety nine +MCM one thousand, nine hundred +MCMI one thousand, nine hundred one +MCMII one thousand, nine hundred two +MCMIII one thousand, nine hundred three +MCMIV one thousand, nine hundred four +MCMV one thousand, nine hundred five +MCMVI one thousand, nine hundred six +MCMVII one thousand, nine hundred seven +MCMVIII one thousand, nine hundred eight +MCMIX one thousand, nine hundred nine +MCMX one thousand, nine hundred ten +MCMXI one thousand, nine hundred eleven +MCMXII one thousand, nine hundred twelve +MCMXIII one thousand, nine hundred thirteen +MCMXIV one thousand, nine hundred fourteen +MCMXV one thousand, nine hundred fifteen +MCMXVI one thousand, nine hundred sixteen +MCMXVII one thousand, nine hundred seventeen +MCMXVIII one thousand, nine hundred eighteen +MCMXIX one thousand, nine hundred nineteen +MCMXX one thousand, nine hundred twenty +MCMXXI one thousand, nine hundred twenty one +MCMXXII one thousand, nine hundred twenty two +MCMXXIII one thousand, nine hundred twenty three +MCMXXIV one thousand, nine hundred twenty four +MCMXXV one thousand, nine hundred twenty five +MCMXXVI one thousand, nine hundred twenty six +MCMXXVII one thousand, nine hundred twenty seven +MCMXXVIII one thousand, nine hundred twenty eight +MCMXXIX one thousand, nine hundred twenty nine +MCMXXX one thousand, nine hundred thirty +MCMXXXI one thousand, nine hundred thirty one +MCMXXXII one thousand, nine hundred thirty two +MCMXXXIII one thousand, nine hundred thirty three +MCMXXXIV one thousand, nine hundred thirty four +MCMXXXV one thousand, nine hundred thirty five +MCMXXXVI one thousand, nine hundred thirty six +MCMXXXVII one thousand, nine hundred thirty seven +MCMXXXVIII one thousand, nine hundred thirty eight +MCMXXXIX one thousand, nine hundred thirty nine +MCMXL one thousand, nine hundred forty +MCMXLI one thousand, nine hundred forty one +MCMXLII one thousand, nine hundred forty two +MCMXLIII one thousand, nine hundred forty three +MCMXLIV one thousand, nine hundred forty four +MCMXLV one thousand, nine hundred forty five +MCMXLVI one thousand, nine hundred forty six +MCMXLVII one thousand, nine hundred forty seven +MCMXLVIII one thousand, nine hundred forty eight +MCMXLIX one thousand, nine hundred forty nine +MCML one thousand, nine hundred fifty +MCMLI one thousand, nine hundred fifty one +MCMLII one thousand, nine hundred fifty two +MCMLIII one thousand, nine hundred fifty three +MCMLIV one thousand, nine hundred fifty four +MCMLV one thousand, nine hundred fifty five +MCMLVI one thousand, nine hundred fifty six +MCMLVII one thousand, nine hundred fifty seven +MCMLVIII one thousand, nine hundred fifty eight +MCMLIX one thousand, nine hundred fifty nine +MCMLX one thousand, nine hundred sixty +MCMLXI one thousand, nine hundred sixty one +MCMLXII one thousand, nine hundred sixty two +MCMLXIII one thousand, nine hundred sixty three +MCMLXIV one thousand, nine hundred sixty four +MCMLXV one thousand, nine hundred sixty five +MCMLXVI one thousand, nine hundred sixty six +MCMLXVII one thousand, nine hundred sixty seven +MCMLXVIII one thousand, nine hundred sixty eight +MCMLXIX one thousand, nine hundred sixty nine +MCMLXX one thousand, nine hundred seventy +MCMLXXI one thousand, nine hundred seventy one +MCMLXXII one thousand, nine hundred seventy two +MCMLXXIII one thousand, nine hundred seventy three +MCMLXXIV one thousand, nine hundred seventy four +MCMLXXV one thousand, nine hundred seventy five +MCMLXXVI one thousand, nine hundred seventy six +MCMLXXVII one thousand, nine hundred seventy seven +MCMLXXVIII one thousand, nine hundred seventy eight +MCMLXXIX one thousand, nine hundred seventy nine +MCMLXXX one thousand, nine hundred eighty +MCMLXXXI one thousand, nine hundred eighty one +MCMLXXXII one thousand, nine hundred eighty two +MCMLXXXIII one thousand, nine hundred eighty three +MCMLXXXIV one thousand, nine hundred eighty four +MCMLXXXV one thousand, nine hundred eighty five +MCMLXXXVI one thousand, nine hundred eighty six +MCMLXXXVII one thousand, nine hundred eighty seven +MCMLXXXVIII one thousand, nine hundred eighty eight +MCMLXXXIX one thousand, nine hundred eighty nine +MCMXC one thousand, nine hundred ninety +MCMXCI one thousand, nine hundred ninety one +MCMXCII one thousand, nine hundred ninety two +MCMXCIII one thousand, nine hundred ninety three +MCMXCIV one thousand, nine hundred ninety four +MCMXCV one thousand, nine hundred ninety five +MCMXCVI one thousand, nine hundred ninety six +MCMXCVII one thousand, nine hundred ninety seven +MCMXCVIII one thousand, nine hundred ninety eight +MCMXCIX one thousand, nine hundred ninety nine +MM two thousand diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/suppletive.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/suppletive.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4718e27bd13433c5376acc1fcf509bfa64edc42d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/suppletive.tsv @@ -0,0 +1,112 @@ +deer +fish +sheep +foot feet +goose geese +man men +mouse mice +tooth teeth +woman women +won +child children +ox oxen +wife wives +wolf wolves +analysis analyses +criterion criteria +lbs +focus foci +percent +hertz +kroner krone +inch inches +calory calories +yen +megahertz +gigahertz +kilohertz +hertz +CC +c c +horsepower +hundredweight +kilogram force kilograms force +mega siemens +revolution per minute revolutions per minute +mile per hour miles per hour +megabit per second megabits per second +square foot square feet +centimeter per second centimeters per second +meter per second meters per second +kilometer per second kilometers per second +meter per hour meters per hour +bit per second bits per second +kilometer per hour kilometers per hour +kilobit per second kilobits per second +kilo bit per second kilo bits per second +megabit per second megabits per second +mega bit per second mega bits per second +gigabit per second gigabits per second +giga bit per second giga bits per second +terabit per second terabits per second +tera bit per second tera bits per second +petabit per second petabits per second +peta bit per second peta bits per second +byte per second bytes per second +kilobyte per second kilobytes per second +kilo byte per second kilo bytes per second +megabyte per second megabytes per second +mega byte per second mega bytes per second +gigabyte per second gigabytes per second +giga byte per second giga bytes per second +terabyte per second terabytes per second +tera byte per second tera bytes per second +petabyte per second petabytes per second +peta byte per second peta bytes per second +degree Celsius degrees Celsius +degree Fahrenheit degrees Fahrenheit +degree celsius degrees celsius +degree fahrenheit degrees fahrenheit +kelvin kelvin +ATM +AU +BQ +CC +CD +DA +EB +EV +F +GB +G +GL +GPA +GY +HA +H +HL +GP +HS +KB +KL +KN +KT +KV +LM +MA +MA +MB +MC +MF +M +MM +MS +MV +MW +PB +PG +PS +S +TB +YB +ZB diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ip_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ip_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..03e2529895d9abd2ece193be4bd354497bd82d68 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ip_prompt.tsv @@ -0,0 +1,2 @@ +IP address is +IP is \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ssn_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ssn_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8bbdb9f752f7739c24b8dcaa075d50454423adad --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/ssn_prompt.tsv @@ -0,0 +1,4 @@ +ssn is SSN is +ssn is SSN is +SSN is +SSN \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/telephone_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/telephone_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6dcfb6cad223f69c6763a53fb745c21a089c340c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/telephone/telephone_prompt.tsv @@ -0,0 +1,5 @@ +call me at +reach at +reached at +my number is +hit me up at \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..026a6a90ccceac64bdfd8ecff7bfba28298331ea --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/suffix.tsv @@ -0,0 +1,12 @@ +p.m. PM +p.m PM +pm PM +P.M. PM +P.M PM +PM PM +a.m. AM +a.m AM +am AM +A.M. AM +A.M AM +AM AM diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/zone.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/zone.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0fda04208c8741d41f7a837b5bc60680b1787229 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/time/zone.tsv @@ -0,0 +1,14 @@ +cst CST +c.s.t CST +cet CET +c.e.t CET +pst PST +p.s.t PST +est EST +e.s.t EST +pt PT +p.t PT +et ET +e.t ET +gmt GMT +g.m.t GMT diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0ee8e2c1d79bf2dd2c16513600e217670821f5b1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives.tsv @@ -0,0 +1,45 @@ +Hon. Honorable +Mr. Mister +Mrs. Misses +Ms. Miss +Mr Mister +Mrs Misses +Ms Miss +AC air conditioning +AC air conditioner +AC air conditioners +AC alternating current +&Co. and Co. +&Co. and Company +Mon Monday +Tu Tuesday +Wed Wednesday +Th Thursday +Thur Thursday +Thurs Thursday +Fri Friday +Sat Saturday +Sun Sunday +Mon Mon +Tu Tu +Wed Wed +Th Th +Thur Thur +Thurs Thurs +Fri Fri +Sat Sat +Sun Sun += equals +# number +No. number +No number +NO number +NO. number +NO nitrogen monoxide +NO NO +NO. NO. +No. No. +No No +VOL Volume +VOL. Volume +TV Television diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives_all_format.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives_all_format.tsv new file mode 100644 index 0000000000000000000000000000000000000000..449195c4f41f0d5fdb13451be58f13ea3eb6cd9b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/alternatives_all_format.tsv @@ -0,0 +1,14 @@ +st street +st saint +dr doctor +dr drive +mt mount +sr senior +prof professor +mt mountain +sr senior +jr junior +vol volume +rd road +ave avenue +approx approximately diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..19577a21ec0d3156304ec94896488e6e9166ea86 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr.tsv @@ -0,0 +1,14713 @@ +Ph.D. p h d +Hon. honorable +& and +Mt. Mount +Maj. Major +Rev. Reverend +# hash +Gov. governor +vs. versus +vs versus +dept. department +vol volume +vol. volume +bldg. building +Bldg. Building +apt. apartment +Apt. Apartment +Σ sigma +η eta +κ kappa +ω omega +σ sigma +α alpha +ν nu +δ delta +ι iota +_ underscore +% percent +& ampersand +* asterisk ++ plus +/ slash += equal sign +^ circumflex +{ left brace +| vertical bar +} right brace +~ tilde +ltd limited +int'l international +$ dollar +A. A. a a +A.A. a a +A.A.A. a a a +A&A a and a +AAAI a a a i +AAAM a a a m +AAAs a a a's +AAAS a a a s +AAAW a a a w +AABA a a b a +AaB a a b +A. A. B. a a b +AAB a a b +AABC a a b c +Aabo a a b o +AABP a a b p +AABW a a b w +aac a a c +AAC a a c +AACAR a a c a r +AACC a a c c +AACCUP a a c c u p +AACMI a a c m i +AACNo a a c n o +AACR a a c r +AACS a a c s +AACSB a a c s b +AACTA a a c t a +AACUPR a a c u p r +A.A.D. a a d +AADT a a d t +AADTs a a d t's +Aadu a a d u +AAEA a a e a +AAE a a e +AAERT a a e r t +AAF a a f +AAFCA a a f c a +AAFC a a f c +AAFC's a a f c's +AAFld a a f l d +AAFPOA a a f p o a +AAGPBL a a g p b l +AAHHE a a h h e +AAI a a i +AAIB a a i b +AAIP a a i p +AAK a a k +Aap a a p +AAP a a p +AAPB a a p b +AAPC a a p c +AAPEP a a p e p +AAPG a a p g +AAPM a a p m +AAPT a a p t +A.A.R.M. a a r m +A.B.A. a b a +A. B. a b +A.B. a b +A.B.A. J. a b a j +A.B.C. a b c +A. B. G. a b g +ABG a b g +ABH a b h +ABHD a b h d +Abhi a b h i +ABK a b k +abl a b l +Abl a b l +ABL a b l +ABLV a b l v +ABM a b m +ABMC a b m c +ABN a b n +ABN's a b n's +Abp a b p +ABP a b p +ABPD a b p d +ABPI a b p i +ABPN a b p n +ABPP a b p p +ABPRS a b p r s +ABPW a b p w +Abr a b r +ABR a b r +abv a b v +ABV a b v +ABW a b w +ABX a b x +Abz a b z +A.C.A. a c a +ACA a c a +A&C a and c +A. C. a c +A.C. a c +acac a c a c +Acad a c a d +ACAD a c a d +A.C.A.P. a c a p +ACAP a c a p +ACAS a c a s +ACAZ a c a z +ACBA a c b a +ACB a c b +ACBL a c b l +ACBS a c b s +ACBSP a c b s p +ACCA a c c a +ACC a c c +ACCC a c c c +ACCJ a c c j +ACC&S a c c and s +ACC's a c c's +ACCS a c c s +ACD a c d +ACDC a c d c +A.C. D.F.C. A.F.C. a c d f c a f c +A.C.E. a c e +ACF a c f +ACFE a c f e +A.C.G. a c g +ACG a c g +ACGIH's a c g i h's +A.C.H. a c h +ACH a c h +ACHP a c h p +ACHR a c h r +ACHRE a c h r e +Achs a c h's +ACHS a c h s +ACIA a c i a +ACI a c i +ACICS a c i c s +ACIGA a c i g a +ACIP a c i p +Acis a c i's +ACIS a c i s +ACLA a c l a +A.C.L. a c l +ACL a c l +A.C.L.N. a c l n +acls a c l s +ACL's a c l's +ACLS a c l s +ACLU a c l u +ACM a c m +ACMI a c m i +ACMs a c m's +ACN a c n +ACO's a c o's +AcpA a c p a +ACP a c p +AcpB a c p b +ACPI a c p i +ACPO a c p o +ACPP a c p p +ACPSEM a c p s e m +acq a c q +A&CR a and c r +ACR a c r +ACRPS a c r p s +ACSA a c s a +ACSAC a c s a c +ACSBR a c s b r +ACSCN a c s c n +ACSEL a c s e l +ACSIA a c s i a +ACSI a c s i +AcSOC a c s o c +ACSR a c s r +ACSS a c s s +ACSS's a c s s's +A.C.T. a c t +A.C.T.A.F.L. a c t a f l +acu a c u +ACU a c u +ACU's a c u's +ACUS a c u s +ACUV a c u v +ACV a c v +ACWA a c w a +AC&W a c and w +ACW a c w +ACWM a c w m +ACWS a c w s +Acy a c y +ADAA a d a a +A&D a and d +Adab a d a b +ADAC a d a c +A. D. a d +A.D. a d +A.D.A.M. adam +AdaSL a d a s l +ADATA a d a t a +ADBAC a d b a c +ADB a d b +ADBGR a d b g r +ADBICA a d b i c a +ADCA a d c a +AdC a d c +A.D.C. a d c +ADC a d c +ADCAF a d c a f +ADCC a d c c +ADC's a d c's +ADCs a d c's +ADCY a d c y +ADDA a d d a +AD&D a d and d +ADF a d f +ADF's a d f's +ADFS a d f s +ADG a d g +ADGB a d g b +ADH a d h +ADHD a d h d +ADHM a d h m +ADHS a d h s +ADIAC a d i a c +AdK a d k +ADK a d k +Adl a d l +ADL a d l +ADLL a d l l +ADLs a d l's +A.D.M.A.C. a d m a c +adm a d m +Adm a d m +ADM a d m +ADMK a d m k +admn a d m n +ADMN a d m n +ADNAC a d n a c +ADNEC a d n e c +ADNs a d n's +AdP a d p +A.D.P. a d p +ADP a d p +ADPCM a d p c m +ADPF a d p f +ADQ a d q +ADQ's a d q's +ADR a d r +ADRC a d r c +ADRP a d r p +ADRs a d r's +ADSB a d s b +ADSL a d s l +ADSR a d s r +ADSRs a d s r's +ADSs a d s's +ADT a d t +ADTG a d t g +Adwa a d w a +ADWC a d w c +ADX a d x +AEA a e a +A&E a and e +A. E. a e +A.E. a e +AEBN a e b n +A.E.C. a e c +AEC a e c +AECL a e c l +AECR a e c r +AEC's a e c's +Aed a e d +AED a e d +AEE a e e +AEF a e f +AEG a e g +A.E.G.I.S. a e g i s +aegte a e g t e +AEHL a e h l +AEIA a e i a +AEI a e i +AEIOU a e i o u +Aeka a e k a +AEK a e k +ael a e l +AEL a e l +Aema a e m a +AEM a e m +aen a e n +AEO a e o +AEPA a e p a +AEP a e p +aere a e r e +AEre a e r e +AErn a e r n +AERN a e r n +Aert a e r t +AESA a e s a +Aes a e's +AES a e s +aet a e t +AET a e t +AETC a e t c +AEU a e u +AEV a e v +AEW a e w +AEX a e x +Afa a f a +AFA a f a +af a f +Af a f +A. F. a f +A.F. a f +AF a f +Afaf a f a f +AFAP a f a p +AFB a f b +AFCA a f c a +afc a f c +A.F.C. a f c +AFC a f c +AFCEC a f c e c +AFCO a f c o +AFC's a f c's +AFCS a f c s +AFCSThe a f c s t h e +Afd a f d +AfD a f d +AfDB a f d b +AFDD a f d d +Afe a f e +AFE a f e +Aff a f f +AFF a f f +AFG a f g +AFGM a f g m +AFH a f h +AFIA a f i a +Afi a f i +AFI a f i +AFIP a f i p +Afiq a f i q +AFI's a f i's +AFJ a f j +A.F.L. a f l +AFL a f l +AFLEG a f l e g +AFLPA a f l p a +AFLP a f l p +AFL's a f l's +AFLUA a f l u a +A. F.M.A. a f m a +AFM a f m +AFMC a f m c +AFMs a f m's +AFN a f n +AFNET a f n e t +AFNOR a f n o r +AfNS a f n s +Afo a f o +AFO a f o +AFOP a f o p +AFP a f p +AFRA a f r a +Afr a f r +AFR a f r +AFRC a f r c +AFRL a f r l +AFROTC a f r o t c +AFRTS a f r t s +AFS a f s +AFSC a f s c +AFSPC a f s p c +A. F. T. a f t +AFTRA a f t r a +AFTRA's a f t r a's +AFTRS a f t r s +AFV a f v +AFW a f w +AFWS a f w s +AFX a f x +A. G. a g +A.G. a g +AGB a g b +AGC a g c +Agco a g c o +AGCO a g c o +Agda a g d a +AGD a g d +A.G.E. a g e +AGF a g f +AG&G a g and g +agg a g g +Agi a g i +AGI a g i +AGID a g i d +AGIR a g i r +Agis a g i's +A.G.J. a g j +A. G. K. a g k +AGL a g l +agli a g l i +AGM a g m +Agn a g n +AgN a g n +AGN a g n +AGNs a g n's +A.G.P. a g p +AGP a g p +AGPL a g p l +AGPS a g p s +AGPW a g p w +AGRs a g r's +AGS a g s +AGSM a g s m +Agsu a g s u +agt a g t +AGT a g t +Agte a g t e +Agu a g u +AGU a g u +AGV a g v +A.H.A. a h a +A. H. a h +A.H. a h +A. H. C. a h c +AHC a h c +AHD a h d +Ahdhra a h d h r a +AHDR a h d r +ahe a h e +Ahe a h e +AHEC a h e c +AHF a h f +Ahirs a h i r's +Ahk a h k +AHL's a h l's +AHP a h p +AHRC a h r c +AHRS a h r s +AHSAA a h s a a +Ahsa'i a h s a i +Ahta a h t a +aht a h t +AHT a h t +AHTD a h t d +Ahu a h u +Ahva a h v a +AHV a h v +AIAA a i a a +A.I.A. a i a +A&I a and i +A. I. a i +A.I. a i +AI a i +AIAP a i a p +Aias a i a's +AIAS a i a s +AIATSIS a i a t s i s +AIAW a i a w +AIAWU a i a w u +A.I.B.A. a i b a +AIB a i b +AIBO's a i b o's +AIBS a i b s +AIC a i c +AICAR a i c a r +AICC a i c c +AICCCR a i c c c r +aici a i c i +Aicme a i c m e +AICN a i c n +AICPA a i c p a +AICP a i c p +AICPS a i c p s +AICs a i c's +AICTE a i c t e +AICUF a i c u f +AIDA a i d a +AIDN a i d n +AIEC a i e c +AIEE a i e e +Aiel a i e l +AIEP a i e p +AIFA a i f a +AIF a i f +AIFB a i f b +Aife a i f e +AIFF a i f f +AIFM a i f m +AIFMD a i f m d +AiG a i g +AIG a i g +Aigis a i g i's +AIHA a i h a +AIH a i h +AIHL a i h l +AIIMS a i i m s +Aija a i j a +Aik a i k +AIK a i k +AIKR a i k r +A.I.M. a i m +AIP a i p +A.I.R. a i r +AIs a i's +AISC a i s c +AISD a i s d +AISRI a i s r i +Aist a i s t +AIST a i s t +AIT's a i t's +Aitu a i t u +Aiud a i u d +aius a i u s +AIVC a i v c +AIW a i w +AIX a i x +A. J. a j +A.J. a j +Ajba a j b a +A.J.B. a j b +AJC a j c +A.J.E. a j e +A.J.G.C. a j g c +A. J. J. a j j +A.J.K. a j k +AJK a j k +AJKF a j k f +AJL a j l +A. J. M. a j m +Ajmi a j m i +AJN a j n +AJO a j o +Ajok a j o k +A. J. P. a j p +A.J.P. a j p +AJPW a j p w +A.J.R. a j r +AJR a j r +Ajsa a j s a +A.J.S. a j s +AJS a j s +A. J. T. a j t +A.J.T. a j t +AJT a j t +Aju a j u +AJUFE a j u f e +AJUSCO a j u s c o +AJV a j v +AJW a j w +AJWRC a j w r c +Akaa a k a a +a.k.a. a k a +aka. a k a +aka a k a +Aka. a k a +Aka a k a +AKA a k a +A&K a and k +A. K. a k +A.K. a k +AK a k +AKAP a k a p +AKAPs a k a p's +akas a k a's +AKAs a k a's +AKB a k b +AKC a k c +AKCR a k c r +AKD a k d +AKG a k g +AKG's a k g's +akh a k h +AKH a k h +AKHS a k h s +AKM a k m +AKN a k n +Akpa a k p a +AKP a k p +AKPD a k p d +AKQA a k q a +AKR a k r +A.K.S. a k s +AKS a k s +Aku a k u +AKU a k u +AKW a k w +A&L a and l +A. L. a l +A.L. a l +ALDF a l d f +ALDH a l d h +ALDS a l d s +ALDT a l d t +ALFASID a l f a s i d +ALFIDI a l f i d i +ALFTP a l f t p +Alh a l h +ALH a l h +A. L. M. a l m +A.L.P. a l p +ALPG a l p g +ALR a l r +ALSC a l s c +ALSF a l s f +ALSP a l s p +ALUs a l u's +A.L.V. a l v +ALVF a l v f +ALW a l w +ALWS a l w s +AMAA a m a a +A'ma a m a +A&M a and m +a.m. a m +a.m a m +A. M. a m +A.M. a m +AMAP a m a p +AMARC a m a r c +AMAs a m a's +amb a m b +Amb a m b +AMB a m b +AmBX a m b x +A.M.C. a m c +AMC a m c +AMC's a m c's +AMCs a m c's +Amda a m d a +Amd a m d +AMD a m d +AMDH a m d h +AMDISA a m d i s a +AMD's a m d's +A.M.E. a m e +AMF a m f +AM&FM a m and f m +A. M. G. a m g +AMG a m g +AMGTV a m g t v +Amha a m h a +AMH a m h +A.M.H.S. a m h s +A. M. J. a m j +AMJ a m j +A. M. K. a m k +A.M.K. a m k +AMK a m k +aml a m l +AML a m l +Amli a m l i +AMLS a m l s +Amlwch a m l w c h +Amm a m m +A. M. M. a m m +AMM a m m +AMMAYI a m m a y i +AMMB a m m b +AMMK a m m k +AMN a m n +AMNH a m n h +AMNRL a m n r l +AM&O a m and o +AMPK a m p k +AMPL a m p l +AMP's a m p's +Ampt a m p t +AMPTP a m p t p +amr a m r +A. M. R. a m r +AMREF a m r e f +AMSA a m s a +A&M's a and m's +Ams' a m's +amsl a m s l +AMSO's a m s o's +AMSRS a m s r s +AMS's a m s's +AMSS a m s s +AMSU a m s u +Amta a m t a +amt a m t +Amt a m t +A.M.T. a m t +AMT a m t +AMTB a m t b +AMTPAT a m t p a t +Amts a m t's +AMTV a m t v +Amu a m u +AMU a m u +AMU's a m u's +A.M.V.M. a m v m +AMW a m w +AMX a m x +AMYF a m y f +A. N. a n +A.N. a n +ANB a n b +ANBO a n b o +ANC a n c +ANCC a n c c +ANC's a n c's +andFHL a n d f h l +ANDOC a n d o c +ANDP a n d p +ANDPOP a n d p o p +andPPP a n d p p p +Anej a n e j +ANELFA a n e l f a +ANF a n f +ANFIS a n f i s +ANFP's a n f p's +ANGB a n g b +ANGPTL a n g p t l +ANGRAU a n g r a u +Angti a n g t i +Angu a n g u +Anhe a n h e +ANI a n i +ANL a n l +ANM a n m +ANP a n p +ANPP a n p p +ANPR a n p r +Anr a n r +ANR a n r +ANRC a n r c +ANREF's a n r e f's +Anrep a n r e p +ANRS a n r s +ANRW a n r w +A.N.S. a n s +ANS a n s +ANSF a n s f +ANTB a n t b +ANTM a n t m +ANTV a n t v +ANUGA a n u g a +ANWB a n w b +ANX a n x +ANZ a n z +A&O a and o +AOA a o a +AOAC a o a c +A. O. a o +A.O. a o +AOB a o b +AOC a o c +AOCB a o c b +AOCCs a o c c's +AOCE a o c e +AOD a o d +Aodh a o d h +Aodla a o d l a +AODV a o d v +AOE a o e +AOGCC a o g c c +Aogo a o g o +AOH a o h +Aoke a o k e +Aoko a o k o +AOKP a o k p +Aola a o l a +Aol a o l +AOL a o l +AOL's a o l's +AOLTV a o l t v +AOM a o m +AOMC a o m c +AONB a o n b +Aone a o n e +Aoni a o n i +Aoos a o o's +AOPA a o p a +AOP a o p +aor a o r +AOR a o r +AORs a o r's +AORS a o r s +aos a o s +AoS a o s +AOS a o s +Aotus a o t u's +AOTW a o t w +aov a o v +aovf a o v f +AOWC a o w c +A&P a and p +A. P. a p +A.P. a p +APBA a p b a +APB a p b +APBL a p b l +APCA a p c a +Apc a p c +APC a p c +APCCR a p c c r +APCh a p c h +APCR a p c r +APCRDA a p c r d a +APC's a p c's +APCs a p c's +APDA a p d a +APD a p d +APDM a p d m +APEGBC a p e g b c +APFA a p f a +APF a p f +APFCT a p f c t +APFOL a p f o l +APFSDS a p f s d s +APFUTU a p f u t u +apg a p g +APG a p g +APGAW a p g a w +Aph a p h +A.P.H. a p h +APH a p h +APHEDA a p h e d a +APHL a p h l +Apiao a p i a o +Api a p i +APi a p i +API a p i +Apic a p i c +APICv a p i c v +API's a p i's +A.P.J. a p j +APJ a p j +Apl a p l +APL a p l +APLP a p l p +APLS a p l s +Aplu a p l u +APM a p m +APML a p m l +APMR a p m r +APMSO a p m s o +APN a p n +A.P.N.C. a p n c +Apphttp a p p h t t p +Appl a p p l +APPO's a p p o's +appr a p p r +Appts a p p t's +appu a p p u +Appu a p p u +APRA's a p r a's +APRC a p r c +APR's a p r's +APRS a p r s +APRST a p r s t +A&P's a and p's +Aps a p's +APs a p's +APS a p s +APSF a p s f +APSL a p s l +APTA a p t a +AP&T a p and t +APW a p w +APX a p x +APXS a p x s +Aqa a q a +AQA a q a +AQC a q c +AQI a q i +Aql a q l +AQP a q p +Aqr a q r +A&R a and r +ARA a r a +A. R. a r +A.R. a r +A.R.B. a r b +ARD a r d +ARDF a r d f +A.R.E. a r e +A.R.F. a r f +ARF a r f +A.R.G. a r g +ArgR a r g r +ARGs a r g's +ARGT a r g t +ArH a r h +ARJ a r j +ARL a r l +ARLFC a r l f c +A. R. M. a r m +ARMv a r m v +Arnd a r n d +ARNG a r n g +ARNT a r n t +A.R.P. a r p +ARQ a r q +ARTL a r t l +A.R.U. a r u +arv a r v +Arv a r v +ARV a r v +ARVD a r v d +ARVN a r v n +ARW a r w +Arwi a r w i +ARWU a r w u +A.S.A. a s a +ASA a s a +As'ad a s a d +Asai a s a i +Asao a s a o +A.s a's +A. S. a s +A.S. a s +ASAS a s a s +Asasp a s a s p +ASAU a s a u +A.S.B. a s b +ASB a s b +ASBDA a s b d a +asbl a s b l +ASBM a s b m +ASBMH a s b m h +ASBO a s b o +A.S.C. a s c +ASC a s c +ASCB a s c b +ASCE a s c e +asci a s c i +Asci a s c i +ASCP a s c p +ASCW a s c w +A.S.D. a s d +ASD a s d +ASDIC a s d i c +Asdis a s d i's +ASE a s e +ASEC a s e c +ASEE a s e e +ASEF a s e f +ASFA a s f a +ASF a s f +Asfi a s f i +ASGA a s g a +ASG a s g +ASGC a s g c +ASGE a s g e +ASGS a s g s +A. S. H. a s h +A.S.I. a s i +ASL a s l +A.S.M. a s m +ASM a s m +ASMD a s m d +ASME a s m e +ASMIK a s m i k +ASML a s m l +ASMPH a s m p h +ASMSU a s m s u +Asn a s n +ASN a s n +Aso a s o +ASO a s o +ASP a s p +ASPCA a s p c a +Aspe a s p e +ASPTS a s p t s +Asr a s r +ASR a s r +Assn a s s n +assoc a s s o c +Assoc a s s o c +ASSPs a s s p's +ASSR a s s r +Asst a s s t +ASSU a s s u +A. S. T. a s t +ASTAT a s t a t +ASTCL a s t c l +Aste a s t e +ASTE a s t e +ASTM a s t m +ASTR a s t r +Astt a s t t +A.S.U. a s u +ASU a s u +A.S.V. a s v +ASV a s v +Aswa a s w a +ASW a s w +ASX a s x +A&T a and t +atac a t a c +Atac a t a c +Atad a t a d +Ata's a t a's +Atas a t a's +ATA's a t a's +A. T. a t +A.T. a t +ATBF a t b f +ATBs a t b's +atc a t c +A.T.C. a t c +ATC a t c +atcc a t c c +ATCC a t c c +ATCDE a t c d e +ATCL a t c l +ATDC a t d c +ATF a t f +Atg a t g +ATG a t g +ATGM a t g m +ath a t h +Ath a t h +ATH a t h +ATHN a t h n +ATIA a t i a +Atia's a t i a's +Ati a t i +ATi a t i +ATI a t i +Atid a t i d +Atiiq a t i i q +Atil a t i l +ATINC a t i n c +ATIP a t i p +Atiq a t i q +Ativ a t i v +ATJ a t j +Atka a t k a +ATK a t k +ATKN a t k n +ATK's a t k's +Atl a t l +ATL a t l +Atli a t l i +ATLY a t l y +A.T.M.A. a t m a +atm a t m +Atm a t m +ATM a t m +ATMs a t m's +ATN a t n +ATO a t o +Atos a t o's +ATO's a t o's +ATOs a t o's +ATOS a t o s +ATP a t p +atpB a t p b +A.T.Q. a t q +ATR a t r +atri a t r i +Atri a t r i +ATRP a t r p +A&T's a and t's +A.T.s a t's +A.T.S. a t s +ATS a t s +ATSC a t s c +ATSDR a t s d r +AT&SF a t and s f +ATSF a t s f +ATSIC a t s i c +AT&T a t and t +attd a t t d +ATTESA a t t e s a +ATTF a t t f +AT&T's a t and t's +ATTS a t t s +ATTWI a t t w i +ATTWX's a t t w x's +ATU's a t u's +atv a t v +ATV a t v +ATV's a t v's +ATVs a t v's +ATW a t w +AtxA a t x a +ATX a t x +ATXN a t x n +A. U. a u +A.U. a u +Aub a u b +Aubl a u b l +AUC a u c +AUVs a u v's +Auw a u w +avab a v a b +AVAC a v a c +A. V. a v +A.V. a v +AVCA a v c a +AVC a v c +AVCHD a v c h d +AVCs a v c's +AVCS a v c s +AVD a v d +avg a v g +AVG a v g +AVK a v k +AVL a v l +A. V. M. a v m +AVM a v m +Avn a v n +AVN a v n +Avo a v o +AvP a v p +AVP a v p +AVR a v r +AVRs a v r's +AVSA a v s a +Avs' a v's +Avs a v's +AVS a v s +AVSM a v s m +AVU a v u +avvo a v v o +AVX a v x +Awa a w a +A. W. A. a w a +AWA a w a +A. W. A. M. a w a m +AWAs a w a's +A. W. a w +A.W. a w +A. W. B. a w b +A.W.B. a w b +AWB a w b +AWB's a w b's +A.W.C. a w c +AWC a w c +AWD a w d +A. W. F. a w f +AWG a w g +AWGIE a w g i e +AWGN a w g n +A. W. H. a w h +AWHL a w h l +AWI a w i +AWM a w m +Awo a w o +AWP a w p +AWP's a w p's +AWR a w r +AWSA a w s a +AWU a w u +Awwa a w w a +AXAF a x a f +A. X. a x +AXS a x s +AXV a x v +A. Y. a y +A.Y. a y +AYF a y f +AYK a y k +AYP a y p +A. Z. a z +A.Z. a z +AZE a z e +Azg a z g +Azi a z i +AZI a z i +AZL a z l +azm a z m +AZN a z n +AZS a z s +AZSTA's a z s t a's +AZT a z t +AzTV a z t v +Baad b a a d +BAAG b a a g +Ba'al b a a l +Baal b a a l +BAAL b a a l +Baam b a a m +Baap b a a p +ba'as b a a's +Baat b a a t +Ba'ath b a a t h +B. A. b a +B.A. b a +B&A b and a +Bac b a c +B.A.C. b a c +BAC b a c +bae b a e +Bae b a e +BAe b a e +BAE b a e +BAFA b a f a +BAF b a f +B.A.R. b a r +BARV b a r v +BASCA b a s c a +BASEC b a s e c +BASF b a s f +B.A.S.P. b a s p +BATVG b a t v g +Bauw b a u w +B'Av b a v +BAV b a v +BBA b b a +BBAG b b a g +B&B b and b +bb b b +B. B. b b +B.B. b b +BB b b +BBB b b b +bbc b b c +B.B.C. b b c +BBC b b c +BBCBBC b b c b b c +BB&CI b b and c i +BBC&PJR b b c and p j r +BBC's b b c's +B.B.D. b b d +BBDO b b d o +BBE b b e +BBF b b f +BBFC b b f c +BBFF b b f f +BBG b b g +BBH b b h +BBI b b i +BBK b b k +BBKL b b k l +BBLB b b l b +BBL b b l +BBM b b m +BBMP b b m p +BBNG b b n g +BBN's b b n's +BBO b b o +BBP b b p +BBQ'er b b q e r +B&Bs b and b's +BB&S b b and s +BBS b b s +BBSes b b s e's +BBS's b b s's +BB&T b b and t +BBT b b t +BBTV b b t v +BBU b b u +BBVA b b v a +BbvCI b b v c i +BBV's b b v's +BBWAA b b w a a +BBWA b b w a +BBWR's b b w r's +BBYA b b y a +BBYO b b y o +BCA b c a +BCAD b c a d +BCAM b c a m +BCATP b c a t p +BCBA b c b a +BCB b c b +bc b c +B. C. b c +B.C. b c +BC b c +BCCA b c c a +BCC b c c +BCCCA b c c c a +Bcci b c c i +BCCI b c c i +BCCP b c c p +BCC's b c c's +BCDA b c d a +BCD b c d +BCEA b c e a +BCE b c e +BCE's b c e's +BCF b c f +BCG b c g +BCG's b c g's +BCHL b c h l +BCHR's b c h r's +BCHS b c h s +BCI b c i +B. C. J. b c j +BCL b c l +BCLR b c l r +BCMA b c m a +BCM b c m +BCMG b c m g +BCMHS b c m h s +BCMS b c m s +BCN b c n +BCN's b c n's +BCP b c p +BCPM b c p m +BCR b c r +BCRF's b c r f's +B.C.'s b c's +BC's b c's +BCs b c's +BCS b c s +BCSC b c s c +BCSic b c s i c +BCSN b c s n +BCT b c t +BCTC b c t c +BCU b c u +BCYP b c y p +BdA b d a +BDA b d a +bd b d +B. D. b d +B.D. b d +BD b d +BDBL b d b l +BDC b d c +BDC's b d c's +BDD b d d +Bde b d e +B. D. E. b d e +BDE b d e +BDF b d f +BDI b d i +BDJ b d j +BDK b d k +BDK's b d k's +BDMI b d m i +BDN b d n +BDNF b d n f +BDO b d o +BDO's b d o's +BDOS b d o s +BDP b d p +BDR b d r +bds b d s +BDS b d s +BDSM b d s m +BdU b d u +BDU b d u +BDV b d v +B. D. W. b d w +B.E.A. b e a +B. E. b e +B.E. b e +Bedw b e d w +B.E.E. b e e +bef b e f +B.E.F. b e f +BEF b e f +BEF's b e f's +bei b e i +Bei b e i +Beih b e i h +BEIR b e i r +Beis b e i's +bej b e j +B.E.M. b e m +BEMs b e m's +BEP b e p +Ber b e r +BER b e r +Bes b e's +B.E.S. b e s +BES b e s +B.F.A. b f a +BFA b f a +bf b f +B. F. b f +B.F. b f +BF b f +BFCA b f c a +BFC b f c +BFC's b f c's +BFDG b f d g +BFES b f e s +BFF b f f +BFFs b f f's +B. F. G. b f g +BFG b f g +BFG's b f g's +BFI b f i +BFI's b f i's +BFJA b f j a +BFKL b f k l +BFL b f l +BFM b f m +BFO b f o +BFRA b f r a +BFR b f r +BFSA b f s a +BFS b f s +BFT b f t +BFU b f u +bfy b f y +BGAB b g a b +BGA b g a +BGB b g b +B. G. b g +B.G. b g +BGC b g c +BGCI b g c i +BGD b g d +BGEA b g e a +B. G. E. b g e +BGH b g h +B. G. J. S. b g j s +BGL b g l +BglII b g l i i +BGM b g m +BGP b g p +BGRA b g r a +BGR b g r +BGRC b g r c +BGRSO b g r s o +bgs b g s +B.G.S. b g s +BGS b g s +BGSU b g s u +BGT b g t +Bgy b g y +Bha b h a +B&H b and h +B. H. b h +B.H. b h +BHCC b h c c +BHDP b h d p +BH&E b h and e +Bhe b h e +BHL b h l +BHMA b h m a +BHMO b h m o +BHMs b h m's +BHMT b h m t +BHO b h o +B. H. P. b h p +BHP b h p +BHRT b h r t +BHS b h s +BHSN b h s n +BHSU b h s u +bhttp b h t t p +BHU b h u +BHUSD b h u s d +BHVS b h v s +B. I. b i +B.I. b i +Bie b i e +BIE b i e +biedt b i e d t +Biem b i e m +B.I.G. b i g +B.I.G.'s b i g's +BiH b i h +BIH b i h +B.I.O.L.A. b i o l a +B. J. b j +B.J. b j +BJCC b j c c +BJCP b j c p +BJD b j d +B.J.F. b j f +B. J. I. b j i +BJJ b j j +BJP b j p +BJP's b j p's +BJPs b j p's +BJPS b j p s +B.J.'s b j's +B.J.T. b j t +BJT b j t +BJU b j u +BJY b j y +B.K.A. b k a +B&K b and k +B. K. b k +B.K. b k +BKCa b k c a +BKC b k c +BKF b k f +BKI b k i +BKN's b k n's +BKO b k o +B.K.R. b k r +BKR b k r +BK's b k's +BKS b k s +BKT b k t +BKTV b k t v +BKV b k v +B. L. b l +B.L. b l +BL b l +BLC b l c +BLCC b l c c +BLCN b l c n +ble b l e +bleg b l e g +blev b l e v +BLG b l g +bli b l i +BLI b l i +Blla b l l a +BLL b l l +BLM b l m +BLMC b l m c +BLOU's b l o u's +BLPP b l p p +BLR b l r +BLRC b l r c +BLR&D b l r and d +B.L.S. b l s +BLS b l s +BLT b l t +BLTF b l t f +BLTs b l t's +BLV b l v +BMAA b m a a +B.M.A. b m a +BMA b m a +BMAL b m a l +BMARC b m a r c +BMA's b m a's +B&M b and m +BMB b m b +B. M. b m +B.M. b m +BMCA b m c a +B.M.C. b m c +BMC b m c +bmd b m d +BMD b m d +BME b m e +BMF b m f +BMG b m g +BMHS b m h s +bmi b m i +BMI b m i +BMIC b m i c +BMIR b m i r +BMI's b m i's +BMIT b m i t +BMJ b m j +BMK b m k +BMKG b m k g +BMMO b m m o +BMNA b m n a +BMNH b m n h +BMNP b m n p +BMO b m o +BMP b m p +BMPs b m p's +BMR b m r +BMRs b m r's +BMSA b m s a +B.M.S. b m s +BMS b m s +bmt b m t +B. M. T. b m t +B.M.T. b m t +BMT b m t +BMT's b m t's +BMU b m u +B.M.V. b m v +BMV b m v +B. M. W. b m w +BMW b m w +BMW M b m w +BMW's b m w's +BMX b m x +BNA b n a +B&N b and n +bnb b n b +BNB b n b +B. N. b n +B.N. b n +BNC b n c +BNCI b n c i +BNCM b n c m +BNCT b n c t +BND b n d +BNDY b n d y +BNET b n e t +BNF b n f +BNFL b n f l +BNI b n i +BNIC b n i c +BNL b n l +BNO b n o +BNP b n p +BNR b n r +BNRC b n r c +BNS b n s +BNSF b n s f +BNST b n s t +BNT b n t +BNU b n u +BNY b n y +BNZ b n z +Boac b o a c +BOAC b o a c +B&O b and o +B. O. b o +B.O. b o +Boc b o c +BoC b o c +BOC b o c +BOCs b o c's +BOCS b o c s +boj b o j +Boj b o j +BOJ b o j +B.O.M.B. b o m b +B&O's b and o's +B.O.S. b o s +BOYZZ b o y z z +BPA b p a +BPB b p b +bp b p +B. P. b p +B.P. b p +BP b p +BPC b p c +BPD b p d +BPER b p e r +B.P.H. b p h +BPI b p i +BPJ b p j +BPK b p k +bpl b p l +BPL b p l +B. P. M. b p m +BPM b p m +BPMN b p m n +B.P.N. b p n +BPN b p n +B.P.O. b p o +BPO b p o +BPPA b p p a +BPP b p p +BPR b p r +B.P.R.D. b p r d +BPRD b p r d +BPSA b p s a +BP's b p's +BPS b p s +BPSK b p s k +BPUP b p u p +BPV b p v +BPY b p y +B&Q b and q +BQB b q b +B. Q. b q +BRAC's b r a c's +BRAM b r a m +BRBDP b r b d p +Brbic b r b i c +B. R. b r +B.R. b r +BRBR b r b r +BRBs b r b's +BRCA b r c a +BRC b r c +BRCC b r c c +BRD b r d +BRF b r f +BRGF b r g f +BRGM b r g m +BRHS b r h s +B.R.I.C.K. b r i c k +BRK b r k +BRL b r l +BRM b r m +BRMS b r m s +BRN b r n +BRNC b r n c +Brne b r n e +Brno b r n o +BRNV b r n v +BRP b r p +BRPS b r p s +BRRA b r r a +BRS b r s +BRSCC's b r s c c's +BRT b r t +BRTs b r t's +BRTS b r t s +BRVM b r v m +BRW b r w +BRWK b r w k +Brza b r z a +BRZ b r z +brzu b r z u +brzy b r z y +Brzyk b r z y k +BSAA b s a a +B.S.A. b s a +BSA b s a +BSAC b s a c +BsaL b s a l +BSBA b s b a +BSB b s b +BSBI b s b i +B. s b's +B.'s b's +Bs b's +B. S. b s +B.S. b s +BS b s +B.S.C. b s c +BSC b s c +BSCL b s c l +bsd b s d +BSD b s d +BSDE b s d e +BSE b s e +BSET b s e t +BSFA b s f a +BSF b s f +BSG b s g +BSH b s h +BSIB b s i b +BSI b s i +BSK b s k +BSL b s l +BSM b s m +BSME b s m e +BSN b s n +BSNL b s n l +BSOG b s o g +BSPA b s p a +BSP b s p +BSPP b s p p +BSR b s r +BSRN b s r n +BSS b s s +BSU b s u +BSV b s v +BSX b s x +BTA b t a +BTAF b t a f +BTB b t b +BTBD b t b d +bt b t +B. T. b t +B.T. b t +BT b t +BTCA b t c a +BTC b t c +BTCC b t c c +BTD b t d +BTF b t f +BTG b t g +BTH b t h +BTK b t k +BTL b t l +BTM b t m +BTN b t n +BTNK b t n k +BTOB b t o b +BTO b t o +BTP b t p +BTRC b t r c +BTRDA b t r d a +Btry b t r y +btsan b t s a n +BT's b t's +BTS b t s +BTT b t t +BTU b t u +btus b t u s +BTUs b t u's +BTV b t v +B.U. b u +B.U.M. b u m +BUV b u v +BVA b v a +BVB b v b +BVB's b v b's +B. V. b v +B.V. b v +BVC b v c +BVFB b v f b +BVI b v i +BVM b v m +BVO b v o +BVR b v r +BVRC b v r c +BVT b v t +BVVL b v v l +Bwa b w a +BWAF b w a f +B&W b and w +BWB b w b +BWBR b w b r +B. W. b w +B.W. b w +B.W.C. b w c +BWC b w c +BWF b w f +BWFC b w f c +BWHBC b w h b c +B. W. I. b w i +BWI b w i +BWIR b w i r +BWO b w o +BWR b w r +BWS b w s +BWT b w t +BWTs b w t's +BWV b w v +BWW b w w +bwwtv b w w t v +Bxa b x a +BX b x +Bxe b x e +BXML b x m l +BYA's b y a's +BYB b y b +B. Y. b y +byc b y c +BYC b y c +BYD b y d +BYFC b y f c +BYG b y g +bygd b y g d +Byk b y k +byn b y n +BYOB b y o b +Byo b y o +BYO b y o +Byou b y o u +BYR b y r +Byrl b y r l +bySLC b y s l c +BYST b y s t +byt b y t +B.Y.U. b y u +BYU b y u +BYU's b y u's +BYX b y x +BYZ b y z +B.Z. b z +BZ b z +BZD b z d +bzhed b z h e d +Bzik b z i k +BZK b z k +BZP b z p +Bzyb b z y b +CAA c a a +CAAC c a a c +CAAC's c a a c's +CAAHEP c a a h e p +CAAM c a a m +CAAS c a a s +CAASE c a a s e +Caat c a a t +CAAT c a a t +C. A. c a +C.A. c a +CA c a +C&A c and a +CAC c a c +CA&CC c a and c c +CACC c a c c +CACM c a c m +Cadw c a d w +CADW c a d w +caeca c a e c a +Cae c a e +CAE c a e +C.A.F.B. c a f b +CAFTT c a f t t +C. A. G. c a g +CAG c a g +CAGM c a g m +C. A. I. c a i +C. A. J. c a j +CALUX c a l u x +CAMC c a m c +CAMLG c a m l g +C.A.M.'s c a m's +C. A. N. c a n +Capt. captain +C. A. R. c a r +cas c a s +Cas c a's +C.A.S. c a s +CAS c a s +Casc c a s c +CASC c a s c +CASD c a s d +CASF c a s f +CASG c a s g +CASQ c a s q +C. A. T. c a t +C.A.T. c a t +CATV c a t v +C.A.W. c a w +CBA c b a +CBBB c b b b +CBB c b b +CBBC c b b c +CBBS c b b s +CBCA c b c a +cb c b +C. B. c b +C.B. c b +CB c b +cbc c b c +CBC c b c +CB&CNS c b and c n s +CBC's c b c's +CBCS c b c s +CBDA c b d a +CBD c b d +CBDs c b d's +C.B.E. c b e +CBE c b e +CBEF c b e f +CBE's c b e's +CBF c b f +CBGB c b g b +CBGB's c b g b's +CBGBs c b g b's +CBG c b g +CBH c b h +CBHG c b h g +CBI c b i +CBI's c b i's +CbiXS c b i x s +CBKB c b k b +CBK c b k +CBKMT c b k m t +CBKRT c b k r t +CBL c b l +CBL's c b l's +CBM c b m +CBMS c b m s +CBN c b n +CBN's c b n's +CBNT c b n t +CBO c b o +CBOT c b o t +CBP c b p +CBPE c b p e +CB&Q c b and q +CBR c b r +CBRE c b r e +CBRN c b r n +CBRNE c b r n e +CBSA c b s a +CBs c b's +CBS c b s +C.B.S.E. c b s e +CBSE c b s e +CBS's c b s's +CBT c b t +CBTC c b t c +CBTU c b t u +CBU c b u +CBUT c b u t +CBV c b v +CBWT c b w t +CBX c b x +CBYT c b y t +CBZH c b z h +'c c +CcaA c c a a +CCAA c c a a +CCAAT c c a a t +cca c c a +Cca c c a +CCA c c a +CCAF c c a f +CCAP c c a p +CCAR c c a r +CCAS c c a s +CCB c c b +CCBCC c c b c c +CCBE c c b e +CCCA c c c a +C&C c and c +C. C. c c +C.C. c c +CCCE c c c e +CCCF c c c f +CCCP c c c p +CCC's c c c's +CCDB c c d b +CCDC c c d c +ccd c c d +CCD c c d +CCDev c c d e v +CCDI c c d i +CCDI's c c d i's +CCDM c c d m +CCDR c c d r +CCD's c c d's +CCDs c c d's +CCEd c c e d +C.C.E.D. c c e d +CCF c c f +CCFO c c f o +CCG c c g +C.C.G.S. c c g s +CCGS c c g s +CCHA c c h a +CCHC c c h c +C.C.H. c c h +CCH c c h +CCHD c c h d +CCHS c c h s +C.C.I.A.A. c c i a a +CCIAA c c i a a +CCi c c i +CCI c c i +CCID c c i d +CCIE c c i e +CCIH c c i h +CCITT c c i t t +CCJ c c j +CCK c c k +CCLC c c l c +CCL c c l +CCMA c c m a +ccm c c m +CCM c c m +CCMM c c m m +CCMP c c m p +CCMS c c m s +CCN c c n +CCNY c c n y +CCO c c o +CCOKC c c o k c +CCOO c c o o +CCOP c c o p +C.C.O.W.E. E.T.F. c c o w e e t f +CCP c c p +CCPD c c p d +CCPL c c p l +CCPN c c p n +CCP's c c p's +CCQ c c q +CCRC c c r c +CCR c c r +CCRCs c c r c's +CCRH c c r h +CCRS c c r s +CCSA c c s a +CCSC c c s c +CC's c c's +CCS c c s +CCSD c c s d +C.C.S.D.N.Y. c c s d n y +C.C.S.M. c c s m +CCSN c c s n +CCSR c c s r +CCS's c c s's +cct c c t +CCT c c t +CCTF c c t f +CCTRN c c t r n +CCTT c c t t +CCTV c c t v +CCUA c c u a +CCU c c u +ccus c c u s +CCV c c v +CCVG c c v g +C. C. W. c c w +CCWD c c w d +CDA c d a +CDAT c d a t +CDB c d b +CDBS c d b s +CDCA c d c a +C&D c and d +cdc c d c +CDC c d c +cd c d +C. D. c d +C.D. c d +CD c d +CDC's c d c's +CDDB c d d b +CD&DR c d and d r +CDE c d e +CDF c d f +CDFI c d f i +CDFW c d f w +CDH c d h +CDi c d i +CDI c d i +CDISC c d i s c +CDI's c d i's +CDK c d k +CDKN c d k n +CDL c d l +CDLI c d l i +CDLS c d l s +CDMA c d m a +CDM c d m +cDNA c d n a +cdnas c d n a s +cDNAs c d n a's +CDN c d n +CDN's c d n's +CDO c d o +CDOs c d o's +CDPC c d p c +CDP c d p +CDP's c d p's +CDPs c d p's +CD&R c d and r +CDR c d r +cds c d s +CD's c d's +CDs c d's +CDS c d s +CDSG c d s g +CDSP c d s p +CDSPCo c d s p c o +CdtA c d t a +CDT c d t +CdTe c d t e +CDTi c d t i +CDTI c d t i +CDT's c d t's +CDTV c d t v +CDTV's c d t v's +CDU c d u +CDU's c d u's +CD&V c d and v +CDV c d v +CDW c d w +CDWP c d w p +Cec c e c +CEC c e c +c'e c e +ce c e +C'e c e +Ce c e +C. E. c e +C.E. c e +CE c e +CECP c e c p +CEDA c e d a +CEDO c e d o +CEDR c e d r +CEDS c e d s +CEEBA c e e b a +CEEB c e e b +CEEH c e e h +Ceel c e e l +CEEOL c e e o l +CEESA c e e s a +CEFC c e f c +CEF c e f +Cefn c e f n +CEFP c e f p +CEFR c e f r +CEGB c e g b +C.E.G. c e g +C. E. H. c e h +CEI c e i +C&EI's c and e i's +C. E. M. c e m +C&EN c and e n +cen c e n +Cen c e n +CEN c e n +ceo c e o +C.E.O. c e o +CEO c e o +Ceol c e o l +CEOP c e o p +Ceorl c e o r l +Ceos c e o's +CEO's c e o's +CEOs' c e o's +CEOs c e o's +C.E.P. c e p +cer c e r +Cer c e r +CER c e r +CERP c e r p +CERPER c e r p e r +CERS c e r s +ces c e s +Ces c e's +C.E.S. c e s +CES c e s +CESL c e s l +CESO c e s o +CETB c e t b +C. E. T. c e t +CEV c e v +CEVCP c e v c p +CEVG c e v g +CEW c e w +Cex c e x +CeX c e x +CEX c e x +CEZ c e z +CEZMS c e z m s +CFAB c f a b +Cfa c f a +CFA c f a +CFAS c f a s +C. F. B. c f b +CFB c f b +CFBG c f b g +CFBISD c f b i s d +CFB's c f b's +C&F c and f +CFC c f c +cf c f +C. F. c f +C.F. c f +CF c f +CFC's c f c's +CFDA c f d a +CFD c f d +C.F.E. c f e +CFE c f e +CFFC c f f c +C. F. F. c f f +CFF c f f +C. F. H. c f h +CFH c f h +CFHS c f h s +CFI c f i +CFIT c f i t +CFJL c f j l +CFJR c f j r +CFL c f l +CFL's c f l's +CFM c f m +CFMEU c f m e u +CFMI c f m i +CFND c f n d +CFNY c f n y +C.F.O.A. c f o a +CFO c f o +CFOP c f o p +CFOs c f o's +CFPI c f p i +CFPL c f p l +CFQ c f q +CFRB c f r b +C.F.R. c f r +CFR c f r +CFRP c f r p +CFS c f s +CFSCI c f s c i +CFSP c f s p +CFTA c f t a +CFT c f t +CFTO c f t o +CFTR c f t r +CFTs c f t's +CFTXOP c f t x o p +CFU c f u +CFYN c f y n +CFZ c f z +CGA c g a +CGAP c g a p +CGB c g b +C&G c and g +CGCFAD c g c f a d +C. G. c g +C.G. c g +CGCRI c g c r i +CGCS c g c s +CGD c g d +CGDK c g d k +CGE c g e +CGF c g f +CGFP c g f p +CGG c g g +CGI c g i +CGIL c g i l +CGKD c g k d +cgl c g l +CGL c g l +CGP c g p +CGPM c g p m +CGPME c g p m e +CGPW c g p w +CGR c g r +CGRP c g r p +CGS c g s +CGSS c g s s +CGT c g t +CGTG c g t g +CGTIC c g t i c +CGU c g u +CGW c g w +CGW's c g w's +C.H.B. c h b +CHBM c h b m +CHBX c h b x +C&H c and h +C.H.C. c h c +C. H. c h +C.H. c h +CHCO c h c o +CHD c h d +CHDK c h d k +CHDS c h d s +CHF c h f +CHFI c h f i +C.H.G. c h g +CHGO c h g o +CHH c h h +CHHOTO c h h o t o +CHHS c h h s +Chhu c h h u +Chirs c h i r's +CHISMS c h i s m s +CHISZ c h i s z +CHK c h k +CHL c h l +CHLR c h l r +CHL's c h l's +C. H. M. c h m +CHM c h m +CHN c h n +CHP c h p +CHP'den c h p d e n +CHPs c h p's +CHPS c h p s +CHR c h r +CHRDI c h r d i +CHRGD c h r g d +C.H.S. c h s +CHS c h s +CHSE c h s e +CHT c h t +CHV c h v +CHX c h x +Chy c h y +CHYR c h y r +CIAC c i a c +Cia c i a +C.I.A. c i a +CIA c i a +CIC c i c +ci c i +Ci c i +C. I. c i +C.I. c i +CI c i +Cicic c i c i c +CICL c i c l +C.I.C.M. c i c m +CICO c i c o +CIC's c i c's +C.I.D. c i d +CIFF c i f f +CIFL c i f l +CIHE c i h e +CIHR c i h r +cii c i i +CII c i i +CIID c i i d +CIJ c i j +CIKM c i k m +C.I.L. c i l +CIL c i l +cim c i m +Cim c i m +CIM c i m +Cio c i o +C.I.O. c i o +CIO c i o +CIOs c i o's +CIOT c i o t +CIPCA c i p c a +Cip c i p +C.I.P. c i p +CIP c i p +CIPD c i p d +CIPFA c i p f a +CIPM c i p m +CISA c i s a +CISC c i s c +cis c i s +CIs c i's +CIS c i s +CITB c i t b +cit c i t +CIT c i t +CiTD c i t d +CITM c i t m +CITN c i t n +CITP c i t p +citS c i t s +CITV c i t v +CITWF c i t w f +CitX c i t x +Ciuc c i u c +Ciu c i u +CiU c i u +CIU c i u +CIUT c i u t +CIVD c i v d +CIW c i w +CIWS c i w s +CIX c i x +Cixi c i x i +CIY c i y +CIZN c i z n +CJA c j a +CJBHL c j b h l +CJCA c j c a +CJCB c j c b +CJC c j c +C. J. c j +C.J. c j +CJD c j d +C. J. E. c j e +CJFL c j f l +CJFT c j f t +CJFX c j f x +CJGC c j g c +CJGO c j g o +CJHL c j h l +CJK c j k +CJL c j l +CJLS c j l s +CJM c j m +CJNT c j n t +CJOC c j o c +CJP c j p +C. J. R. c j r +C.J.R. c j r +CJR c j r +CJSC c j s c +C. J. S. c j s +CJS c j s +CJT c j t +CJUF c j u f +CJWC c j w c +CKAC c k a c +CKC c k c +C. K. c k +C.K. c k +CK c k +CKD c k d +CKE c k e +CKLM c k l m +CKLQ c k l q +CKLW c k l w +CKS c k s +CKSO c k s o +CKVL c k v l +C. K. W. c k w +CKWK c k w k +ckx c k x +CKXX c k x x +CKY c k y +CLAC c l a c +CLA c l a +CLAS c l a s +CLBs c l b's +clc c l c +CLC c l c +cl c l +C. L. c l +C.L. c l +CL c l +CLCN c l c n +CLD c l d +Cle c l e +CLE c l e +CLF c l f +C.L.G. c l g +CLG c l g +CLHIA c l h i a +CLHS c l h s +CLI c l i +CLK c l k +cllr c l l r +CLM c l m +CLMD c l m d +CLMP c l m p +CL&N c l and n +Clo c l o +CL&P c l and p +CLP c l p +CLRC c l r c +C. L. R. c l r +C.L.R. c l r +CLR c l r +CLRK c l r k +CLSA c l s a +CLSC c l s c +CLs c l's +CLS c l s +CLT c l t +CLTD c l t d +CLTPA c l t p a +CLTs c l t's +Clu c l u +CLU c l u +CLUK c l u k +CLV c l v +Clwyd c l w y d +CLX c l x +C&MA c and m a +CMAC c m a c +C.M.A. c m a +CMA c m a +CMAL c m a l +CMAP's c m a p's +CMAS c m a s +CMAT c m a t +C.M.B. c m b +CMB c m b +CMCC c m c c +CMCCDI c m c c d i +C.M.C. c m c +CMC c m c +cm c m +C. M. c m +C.M. c m +CM c m +CMCs c m c's +CMCS c m c s +CMD c m d +Cmde c m d e +CMDISE c m d i s e +cmdr c m d r +CMEC c m e c +CME c m e +CMF c m f +CMFS c m f s +C. M. G. c m g +C.M.G. c m g +CMG c m g +CMHC c m h c +CMH c m h +CMHR c m h r +CMI c m i +CMJ c m j +CMLA c m l a +cml c m l +C.M.L. c m l +CML c m l +CMLDP c m l d p +CMLL c m l l +CMLL's c m l l's +CMM c m m +CMMI c m m i +CMMs c m m's +cmn c m n +CMO c m o +CMO's c m o's +CMPC c m p c +CMP c m p +CMPS c m p s +CMPSO c m p s o +CMPV c m p v +CMQ c m q +CMQT c m q t +C. M. R. c m r +CMR c m r +CMSAF c m s a f +C.M.S. c m s +CMS c m s +C.M.S.L. c m s l +CMT c m t +Cmte c m t e +CMT's c m t's +CMTU c m t u +CMUCL c m u c l +CMU c m u +CMU's c m u's +CMV c m v +CMVM c m v m +CMVSS c m v s s +CMWF c m w f +CMXXII c m x x i i +CMYK c m y k +CNA c n a +CNBC c n b c +CNB c n b +CNC c n c +cn c n +C. N. c n +C.N. c n +CN c n +CNC's c n c's +CND c n d +CNDL c n d l +CNEB c n e b +CNE c n e +CNF c n f +CNFE c n f e +CNFK c n f k +CNG c n g +CNI c n i +CNJ c n j +CNJF c n j f +CNK c n k +C. N. L. c n l +C.N.L. c n l +CNL c n l +CNMs c n m's +CNN c n n +CNO c n o +CNPC c n p c +CNP c n p +cnr c n r +CNR c n r +CNRMA c n r m a +CNRS c n r s +CNSA c n s a +CNs c n's +CNS c n s +CNSS c n s s +CNSW c n s w +CNTA c n t a +CNT c n t +CNTE c n t e +CNTI c n t i +CNTK c n t k +CNTN c n t n +CNTs c n t's +CNTS c n t s +CNTV c n t v +CNU c n u +Cnut's c n u t's +CNV c n v +CNVs c n v's +CNW c n w +CNWS c n w s +CNZ c n z +COA c o a +C&O c and o +C. O. c o +C.O. c o +Co. company +C.O.D. c o d +Col. colonel +C.O.M.L. c o m l +CPAC c p a c +CPA c p a +CPA's c p a's +CPB c p b +CPBL c p b l +C&P c and p +CPC c p c +cp c p +C. P. c p +C.P. c p +CP c p +CPC's c p c's +CPCs c p c's +CPCS c p c s +CPD c p d +CPDM c p d m +CPD's c p d's +CPEB c p e b +CPEC c p e c +C.P.E. c p e +CPE c p e +CPEO c p e o +CPF c p f +CPFO c p f o +CPG c p g +CPHC c p h c +CPH c p h +CPI c p i +CPIFL c p i f l +CPIM c p i m +CPI's c p i's +CPJ c p j +CPL c p l +CPLP c p l p +CPM c p m +CPN c p n +CPNI c p n i +cpo c p o +C.P.O. c p o +CPO c p o +CPOE c p o e +CPPA c p p a +CPPCC c p p c c +CPP c p p +CPPE c p p e +CPPIB c p p i b +CPPM c p p m +CPQ c p q +CPR c p r +CPRS c p r s +CPSA c p s a +CPSC c p s c +CPS c p s +CPSF c p s f +CPSK c p s k +CPSL c p s l +CPSU c p s u +CPTA c p t a +CPT c p t +CPTM c p t m +CPUC c p u c +CPU c p u +CPUID c p u i d +CPUSA c p u s a +CPUs c p u's +CPUT c p u t +CPVA c p v a +CPV c p v +CPW c p w +CQAC c q a c +CQBR c q b r +CQC c q c +cq c q +CQ c q +CQD c q d +CQDs c q d's +CQM c q m +CQO c q o +CQU c q u +C. R. A. c r a +CRA c r a +C.R.A.Z.Y. c r a z y +CRBC c r b c +CRC c r c +cr c r +C. R. c r +C.R. c r +CR c r +Cre c r e +CRE c r e +CRF c r f +CRG c r g +CRHK c r h k +CRH's c r h's +CRHS c r h s +CRJ c r j +crk c r k +Crkva c r k v a +crkve c r k v e +CRL c r l +CRLs c r l's +crm c r m +CRM c r m +CRNA c r n a +CRN c r n +Crne c r n e +Crni c r n i +Crno c r n o +CRP c r p +CRPF c r p f +CRRF c r r f +CRs c r's +CRS c r s +CRSP c r s p +CRST c r s t +CRTC c r t c +CRT c r t +CRTC's c r t c's +CRUA c r u a +CRWRC c r w r c +CRW's c r w's +CRX c r x +CSAC c s a c +csa c s a +Csa c s a +C.S.A. c s a +CSA c s a +CSAH c s a h +Csak c s a k +Csaky c s a k y +CSAR c s a r +CSAT c s a t +CSAV c s a v +CSB c s b +C&S c and s +C.S.C. c s c +CSC c s c +CSCD c s c d +CSCE c s c e +CSCR c s c r +cs c s +C.s c's +Cs c's +C. S. c s +C.S. c s +CS c s +CSC's c s c's +CSDA c s d a +CSDB c s d b +CSD c s d +CSDL c s d l +CSDN c s d n +CSDP c s d p +CSD's c s d's +CSDS c s d s +CSE c s e +C&SF c and s f +CSF c s f +CSFL c s f l +CSFSO c s f s o +CSGA c s g a +csg c s g +CSG c s g +CSG's c s g's +CSH c s h +CSHL c s h l +CSIC c s i c +CsI c s i +CSI c s i +CSIDC c s i d c +CSILUS c s i l u s +CSIO c s i o +CSIR c s i r +CSIRO c s i r o +CSis c s i's +CSIs c s i's +CSIS c s i s +CSIU c s i u +CSKA c s k a +CSKA's c s k a's +CSK c s k +CSK's c s k's +CSL c s l +CSLI c s l i +CSM c s m +CSNB c s n b +CSNBX c s n b x +CSN c s n +CSN's c s n's +CSOB c s o b +CSO c s o +CSO's c s o's +CSOs c s o's +cspA c s p a +csp c s p +CSP c s p +CSPH c s p h +CSPI c s p i +CSPRA c s p r a +CSPs c s p's +CSRA c s r a +CSR c s r +CSRS c s r s +CSRT c s r t +CSRTT c s r t t +CSSA c s s a +CSS c s s +CSSD c s s d +CSSE c s s e +CSSL c s s l +CSSNCs c s s n c's +CSSR c s s r +CSSS c s s s +CSST c s s t +CSTB c s t b +CST c s t +CSTF c s t f +CSTI c s t i +CSTS c s t s +CSUB c s u b +CSU c s u +CSU's c s u's +Csuz c s u z +CSV c s v +CSW c s w +CSWF c s w f +CSWIP c s w i p +CSWS c s w s +CSX c s x +CSX's c s x's +CSXT c s x t +C.T.A. c t a +CTA c t a +CTAF c t a f +CTARL c t a r l +CTA's c t a's +CTAs c t a's +CTBT c t b t +CTBTO c t b t o +C&T c and t +CTC c t c +ct c t +C. T. c t +C.T. c t +CT c t +CTD c t d +cte c t e +CTE c t e +CTEQ c t e q +CTF c t f +CTGF c t g f +CTI c t i +CTLA c t l a +CTL c t l +CTLLS c t l l s +CTLs c t l's +CTM c t m +CTN c t n +CTO c t o +CTP c t p +ctr c t r +CTRG c t r g +Ctrip c t r i p +CTRL c t r l +CTRP c t r p +CTR's c t r's +CTS c t s +CTSD c t s d +CTSF c t s f +CTSG c t s g +CTUC c t u c +CTU c t u +CTUG c t u g +CTU's c t u's +CTVA c t v a +ctv c t v +CTV c t v +CTV's c t v's +CTX c t x +CTZ c t z +cu c u +C. U. c u +CU c u +C.U.L. c u l +C. U. S. c u s +C. U. T. c u t +CUW c u w +Cuyp c u y p +CVA c v a +CVAK c v a k +CVARG c v a r g +CVB c v b +C&V c and v +CVC c v c +cvcp c v c p +'cv c v +cv c v +C. V. c v +C.V. c v +CV c v +CVD c v d +CVEs c v e's +CVG c v g +CVI c v i +CVJM c v j m +CVM c v m +C.V.N. c v n +CVN c v n +C.V.O. c v o +CVO c v o +CVODE c v o d e +CVOs c v o's +CVP c v p +CVR c v r +CVs c v's +CVS c v s +CVSNT c v s n t +CVT c v t +CVTs c v t's +CVV c v v +CVVHDF c v v h d f +CVW c v w +CVZ c v z +CWA c w a +CWBI c w b i +CWBL c w b l +CWC c w c +C. W. c w +C.W. c w +CWD c w d +CWDS c w d s +C. W. E. c w e +CWE c w e +Cwej c w e j +Cwele c w e l e +CWF c w f +CWGC c w g c +CWG c w g +CWI c w i +C. W. K. c w k +cwm c w m +CWM c w m +CWO c w o +CWPA c w p a +CWR c w r +cwrt c w r t +CWRU c w r u +CWSAC c w s a c +CW's c w's +CWS c w s +CWTS c w t s +CWUR c w u r +C. W. Y. c w y +CWYFL c w y f l +CXC c x c +CXCL c x c l +CXCR c x c r +C.X. c x +C. Y. c y +C.Y. c y +cyl c y l +Cyn c y n +CYOG c y o g +CyP c y p +CYP c y p +Cyrl c y r l +Cys c y's +CYS c y s +CYSK c y s k +CYSY c y s y +czci c z c i +cz c z +C.Z. c z +CZ c z +CZE c z e +CZM c z m +CZW c z w +czy c z y +DAAD d a a d +Daai d a a i +D. A. d a +D.A. d a +D&AD d and a d +D. A. H. d a h +D.A.M. d a m +D.A.N.C.E. dance +D. A. T. d a t +Dav d a v +D.A.V. d a v +DAV d a v +DAV's d a v's +dBa d b a +DBA d b a +DBC d b c +DBCS d b c s +db d b +D. B. d b +D.B. d b +DB d b +DBE d b e +DBI d b i +DBL d b l +DBLE d b l e +DBLP d b l p +DBM d b m +DBMS d b m s +DBR d b r +DB's d b's +D. B. S. d b s +DBS d b s +DBU d b u +DBYC d b y c +DBZ d b z +DCA d c a +DCAF d c a f +DCAM d c a m +DCB d c b +DCCC d c c c +DCCCD d c c c d +DCC d c c +DCCG d c c g +DCCs d c c's +D&C d and c +dc d c +D. C. d c +D.C. d c +DC d c +DCE d c e +DCFC d c f c +DCF d c f +DCG d c g +DCH d c h +DCI d c i +DCK d c k +D.C.L. d c l +DCL d c l +DCMA d c m a +DCM d c m +DCMF d c m f +DCMI d c m i +DCMS d c m s +DCO d c o +DCom d c o m +DCOM d c o m +DCP d c p +DCR d c r +D.C.'s d c's +DC's d c's +DCs d c's +DCS d c s +DCSD d c s d +DCSO d c s o +DCSPER d c s p e r +DCSS d c s s +DCTA d c t a +DCT d c t +DCU d c u +DCUO d c u o +DCUs d c u's +DDA d d a +DDB d d b +D.D.C. d d c +DDC d d c +D&D d and d +D. D. d d +D.D. d d +DDF d d f +DDG d d g +DDHH d d h h +DDi d d i +DDI d d i +DDK d d k +DDLJ d d l j +DDM d d m +DDMS d d m s +DDN d d n +DDNOS d d n o s +Ddoc d d o c +DDO d d o +DDP d d p +DDR d d r +DDRMAX d d r m a x +DDSB d d s b +D.D.S. d d s +DDS d d s +DDT d d t +DDTV d d t v +Ddu d d u +DDWFTTW d d w f t t w +DDX d d x +D.E.A.F. d e a f +Deah d e a h +DEA's d e a's +DECC d e c c +DEC d e c +DECS d e c s +DEDD d e d d +D. E. d e +D.E. d e +deg d e g +D. E. G. d e g +DEG d e g +DEH d e h +DEHP d e h p +dei d e i +Dei d e i +DeI d e i +DEI d e i +Deijm d e i j m +Deip d e i p +D. E. J. d e j +D.E.R. d e r +dez d e z +Dez d e z +Dfa d f a +DFA d f a +DFB d f b +D.F.C. d f c +DFC d f c +DFCs d f c's +D. F. d f +D.F. d f +DFD's d f d's +DfE d f e +D.F.E. d f e +DFE d f e +DfES d f e s +DFF d f f +DFG d f g +DFJ d f j +DFL d f l +DFM d f m +DFOA d f o a +DFO d f o +DFP d f p +DFPs d f p's +DFRC d f r c +DFR d f r +DFs d f's +DFS d f s +DFT d f t +DFW d f w +dfx d f x +DFX d f x +DGAC d g a c +DGA d g a +DGALS d g a l s +DGAP d g a p +DGB d g b +DGCA d g c a +DGC d g c +DGCR d g c r +D&G d and g +D. G. d g +D.G. d g +D. G. E. d g e +DGE d g e +DGERT d g e r t +DGF d g f +DGFI d g f i +DGH d g h +DGK d g k +D. G. L. R. d g l r +DGM d g m +DGMO d g m o +DGP d g p +DGPS d g p s +DGR d g r +Dha d h a +DHA d h a +DHB d h b +DHBs d h b's +DHCP d h c p +D&H d and h +dh d h +D. H. d h +D.H. d h +DH d h +DHEA d h e a +d'HEC d h e c +dhe d h e +DHE d h e +Dheku d h e k u +DHFS d h f s +DHHC d h h c +DHH d h h +DHL d h l +DHMO d h m o +DHNC d h n c +D.H.N. d h n +DHNS d h n s +Dhod d h o d +dhol d h o l +Dhol d h o l +Dhoo d h o o +Dhor d h o r +Dhou d h o u +DHP d h p +DHPH d h p h +DHQ d h q +dhr d h r +D.H.R. d h r +DHRS d h r s +DHS d h s +DHSs d h s's +DHSS d h s s +DHT d h t +DHV d h v +DHW d h w +DHX d h x +D.I. d i +DIFC d i f c +DISD d i s d +DIW d i w +DIX d i x +diy d i y +DiY d i y +D.I.Y. d i y +DIY d i y +Diyn d i y n +DIYs d i y's +D. J. A. d j a +Djam d j a m +dj d j +D. J. d j +D.J. d j +DJ d j +DJ'ed d j e d +Djem d j e m +D.J.H. d j h +Dji d j i +DJI d j i +D.J.N. d j n +Djo d j o +D'Jok d j o k +djr d j r +DJR d j r +D.J.s d j's +DJ's d j's +DJs d j's +D. J. S. d j s +D.J.T. d j t +DjVu d j v u +DJVU d j v u +DjVus d j v u's +D. J. Y. d j y +DKC d k c +D. K. d k +D.K. d k +DKI d k i +DKIM d k i m +DKK d k k +DKNY d k n y +DKPP d k p p +D. K. R. d k r +DKW d k w +dla d l a +DLA d l a +DLC d l c +DLD d l d +dl d l +D. L. d l +D.L. d l +DL d l +DLE d l e +D.L.F. d l f +DLF d l f +DLFN d l f n +DLG d l g +D. L. H. d l h +dlia d l i a +DLI d l i +DLJ d l j +DLJ's d l j's +DLK d l k +DLL d l l +DL&LR d l and l r +DLL's d l l's +DLLs d l l's +DLM d l m +DLMPST d l m p s t +dlo d l o +DLP d l p +DLPFC d l p f c +DLP's d l p's +DLR d l r +DLS d l s +DLSS d l s s +DLSU d l s u +DLSU's d l s u's +DLT d l t +DL&W d l and w +DLZ d l z +DMAA d m a a +DMA d m a +DMB d m b +DMBT d m b t +DMCA d m c a +DMCC d m c c +DMC d m c +DMCM d m c m +D.M.C.'s d m c's +D&M d and m +DMD d m d +DMDK d m d k +D. M. d m +D.M. d m +DM d m +DMDS d m d s +DMed d m e d +DM&E d m and e +DME d m e +DMF d m f +DMG d m g +DMGT d m g t +DMHS d m h s +DMI d m i +DMK d m k +DML d m l +DMN d m n +DMO d m o +DMOFD d m o f d +DMOZ d m o z +DMP d m p +DMRC d m r c +DMR d m r +DMRG d m r g +DMs d m's +DMS d m s +DMSII d m s i i +DMT d m t +DMTF d m t f +DMU d m u +DMUs d m u's +DMV d m v +DMX d m x +DMZ d m z +Dna d n a +DnA d n a +DNA d n a +DNADTC d n a d t c +DNAITV d n a i t v +DNAJA d n a j a +DNAJC d n a j c +DnaJ d n a j +DNAN d n a n +DNB d n b +DNBY d n b y +DNC d n c +DNCG d n c g +DND d n d +D. N. d n +D.N. d n +Dnepr d n e p r +dnes d n e s +Dnes d n e's +DNES d n e s +DNF d n f +DNF's d n f's +DNFs d n f's +DNG d n g +DNJO d n j o +DNKA d n k a +DNK d n k +DNM d n m +DNMT d n m t +DNP d n p +DNQ d n q +DNR d n r +DNSBLs d n s b l's +dns d n s +D.N.S. d n s +DNS d n s +DNSS d n s s +DNT d n t +DNU d n u +DNVP d n v p +D.O.A. d o a +D.O.A.'s d o a's +DOAS d o a s +D.O.B. d o b +D.O.C. d o c +DOCG d o c g +dod d o d +Dod d o d +DoD d o d +DOD d o d +D. O. d o +D.O. d o +DOD's d o d's +D.O.E. d o e +DOGNY d o g n y +DOHC d o h c +D'oh d o h +DOJ d o j +DOJ's d o j's +dok d o k +Dok d o k +domt d o m t +Domt d o m t +Dop d o p +DOP d o p +DOPR d o p r +DOSAAF d o s a a f +DOTC d o t c +dotCMS d o t c m s +DOTD d o t d +DOTP d o t p +DotSVN d o t s v n +Douw d o u w +DOW d o w +DOXO d o x o +dozd d o z d +d'OZ d o z +Dozhd d o z h d +DPAA's d p a a's +DPA d p a +DPB d p b +DPD d p d +dp d p +D. P. d p +D.P. d p +DP d p +DPG d p g +DPJ d p j +DPMA d p m a +DPM d p m +DPN d p n +DPNM d p n m +DPO d p o +dpon d p o n +DPP d p p +DPP's d p p's +DPPX d p p x +DPR d p r +DPRK d p r k +DPRK's d p r k's +DProf d p r o f +DPRP d p r p +DPRT d p r t +DPs d p's +DPS d p s +DPSS d p s s +DPSSL d p s s l +DPT d p t +DPW d p w +DPW's d p w's +DPX d p x +DPYS d p y s +D.Q. d q +DQOY d q o y +DQV d q v +DRB d r b +DRBG d r b g +DRB's d r b's +DRCA d r c a +drc d r c +DRC d r c +DRDA d r d a +DRDC d r d c +DRD d r d +Dr. doctor +D. R. d r +D.R. d r +DRG d r g +dri d r i +Dri d r i +DRI d r i +DRIs d r i's +DRITTES d r i t t e s +Driu d r i u +Driv d r i v +DRJTBC d r j t b c +DRK d r k +DRL d r l +DRLR d r l r +DRMC d r m c +D.R.M. d r m +DRM d r m +Drnis d r n i's +Dro d r o +DRO d r o +DRP d r p +Drs. doctors +DR's d r's +DRS d r s +Drska d r s k a +DRT d r t +druj d r u j +DSA d s a +DsbA d s b a +DS&BB d s and b b +DSB d s b +DSBL d s b l +D.S.C. d s c +DSC d s c +D.S.C.H. d s c h +DSCM d s c m +DSCR d s c r +DSCS d s c s +DSDB d s d b +DSD d s d +DSDM d s d m +dsDNA d s d n a +DSDP d s d p +ds d s +Ds d's +D. S. d s +D.S. d s +DS d s +DSDS d s d s +DSG d s g +DSGi d s g i +DSIC d s i c +dsi d s i +DSi d s i +DSI d s i +DSIF d s i f +DSIR d s i r +DSJ d s j +dsl d s l +D. S. L. d s l +DSL d s l +DSLR d s l r +DSMB d s m b +D.S.M. d s m +DSM d s m +DSMs d s m's +dsn d s n +D&SNGRR d and s n g r r +DSNP d s n p +D.S.O. d s o +DSO d s o +DSPA d s p a +DSP d s p +DSR d s r +DS's d s's +DSS d s s +DST d s t +dsu' d s u +DSU d s u +DSV d s v +DSW d s w +DSX d s x +DSZG d s z g +D.T.A. d t a +DTA d t a +DTC d t c +DTC's d t c's +DTCs d t c's +DTDC d t d c +DTD d t d +DTDs d t d's +D. T. d t +D.T. d t +DT&E d t and e +DTE d t e +DTEK d t e k +DTES d t e s +DTG d t g +DTH d t h +DTIC d t i c +DT&I d t and i +DTI d t i +DTLA d t l a +dtl d t l +DTL d t l +DTLLS d t l l s +DTLS d t l s +DTM d t m +DTO d t o +DTPA d t p a +DTPC d t p c +DTRA d t r a +DTR d t r +DTRS d t r s +DTs d t's +DTS d t s +DTT d t t +D.T.U. d t u +DTU d t u +DTV d t v +dty d t y +D.U. d u +dva d v a +Dva d v a +DVA d v a +DVB d v b +DVBE d v b e +DVBIC d v b i c +DVC d v c +DVCs d v c's +DVCS d v c s +dvd d v d +DVD d v d +DVD's d v d's +DVDs d v d's +dv d v +D. V. d v +D.V. d v +DV d v +dve d v e +dvfb d v f b +DVFS d v f s +DVGS d v g s +DVHS d v h s +dvi d v i +DVI d v i +dvije d v i j e +Dvin d v i n +Dvir d v i r +DVLA d v l a +DVLA's d v l a's +DVL d v l +DVM d v m +DVP d v p +DVR d v r +DVRJRBC d v r j r b c +DVR's d v r's +DVSC d v s c +DVS d v s +DVT d v t +DVV d v v +DVVT d v v t +DVWK d v w k +Dwa d w a +DWA d w a +D&W d and w +D. W. d w +D.W. d w +Dwedw d w e d w +DWET d w e t +DWF d w f +DWG d w g +D.W.H. d w h +Dwi d w i +DWI d w i +DWM d w m +DWN d w n +DWNU d w n u +DWP d w p +dwr d w r +DWR d w r +DWRFC d w r f c +DWS d w s +DWT d w t +DWTS d w t s +dwur d w u r +dxa d x a +DXA d x a +dx d x +D.X. d x +DX d x +DXED d x e d +DXers d x e r's +DXII d x i i +DXK d x k +DXR d x r +DXZ d x z +Dydd d y d d +dy d y +Dy d y +D. Y. d y +DY d y +DYFI d y f i +DYIS d y i s +DYNC d y n c +dypl d y p l +dyr d y r +DYSP d y s p +DZBB d z b b +DZB d z b +dz d z +DZEC d z e c +DZEM d z e m +dzis d z i s +DZNE d z n e +Dzog d z o g +Dzor d z o r +DZRH d z r h +DZRH's d z r h's +DZS d z s +Dzus d z u's +DZXL d z x l +DZZ d z z +EAAB e a a b +EAA e a a +EAAP e a a p +E. A. C. e a c +EAC e a c +EACEF e a c e f +EACS e a c s +EAD e a d +E. A. e a +E.A. e a +EAF e a f +EAGE e a g e +E.A.H. e a h +EAH e a h +EAI e a i +EAIE e a i e +E. A. J. e a j +EAMs e a m's +EAP e a p +E.A.R.L. e a r l +E.A.R.T.H. e a r t h +EASA e a s a +EASB e a s b +eas e a s +Eas e a's +EA's e a's +EAS e a s +Eav e a v +EAX e a x +E. B. e b +E.B. e b +EBL e b l +Ebn e b n +EBNF e b n f +EBOV e b o v +E&BR e and b r +EBR e b r +Ebru e b r u +EBS e b s +EBSL e b s l +EBT e b t +EBU e b u +EBV e b v +E.B.W. e b w +EBWs e b w's +EBZR e b z r +ECAC e c a c +ECA e c a +ECAR e c a r +ECB e c b +ECBJ e c b j +ECBL e c b l +ECCA e c c a +ECCC e c c c +ecc e c c +ECC e c c +ECCI e c c i +Eccl e c c l +ECCRFA e c c r f a +ECCW e c c w +ECD e c d +ec e c +Ec e c +E. C. e c +E.C. e c +EC e c +ece e c e +Ece e c e +ECE e c e +Ecem e c e m +ECFA e c f a +ECF e c f +ECFR e c f r +ECFU e c f u +ecg e c g +ECG e c g +ECH e c h +ECHL e c h l +ECHR e c h r +ECI e c i +ECJ e c j +ECLAC e c l a c +ECLA e c l a +ECLC e c l c +ECL e c l +ECLI e c l i +ECLSS e c l s s +ecma e c m a +ECMA e c m a +ECMC e c m c +ECMDB e c m d b +E.C.M. e c m +ECM e c m +ECMI e c m i +ECML e c m l +ECMLRUS e c m l r u s +ECMWF e c m w f +ECN e c n +ECNP e c n p +ECNs e c n's +ECNS e c n s +ECNZ e c n z +ECP&DA's e c p and d a's +ECP e c p +ECPR e c p r +ECQ e c q +ECR e c r +eCRM e c r m +ECRR e c r r +ECR's e c r's +ECSA e c s a +EC's e c's +E. C. S. e c s +E.C.S. e c s +ECS e c s +ect e c t +E. C. T. e c t +ECT e c t +ECTLO e c t l o +ECTN e c t n +ECTS e c t s +ECV e c v +ECVs e c v's +ECWA e c w a +ECW e c w +edb e d b +EDB e d b +EDC e d c +E. D. e d +E.D. e d +EDF e d f +Edh e d h +EDH e d h +edhe e d h e +EDHF e d h f +EDIB e d i b +EDI e d i +EDK e d k +EDL e d l +EDMA e d m a +EDM e d m +EDMs e d m's +EDMW e d m w +Edmx e d m x +edn e d n +Edn e d n +EDN e d n +EDP e d p +EDQH e d q h +E.D.R. e d r +EDRP e d r p +EDSA e d s a +EDSC e d s c +eds e d s +Eds e d's +E. D. S. e d s +EDS e d s +EDSP e d s p +EDTA e d t a +EDT e d t +EDVAC e d v a c +EDWDO e d w d o +Edw e d w +EDXRD e d x r d +EEA e e a +EEAF e e a f +EEAS e e a s +EEBC e e b c +Eeb e e b +EEC e e c +EECS e e c s +E. E. e e +E.E. e e +Eef e e f +EEF e e f +Eega e e g a +EEI e e i +EEI's e e i's +EEJF e e j f +EELC e e l c +eene e e n e +EEOC e e o c +EEPCO e e p c o +eep e e p +EEP e e p +EERC e e r c +EERI e e r i +Ees e e's +EES e e s +EEST e e s t +EEUU e e u u +eeuw e e u w +Eeuw e e u w +EEZs e e z's +EFA e f a +EFAS e f a s +EFCC e f c c +efc e f c +EFC e f c +EFCJ e f c j +EFD e f d +ef e f +Ef e f +E. F. e f +E.F. e f +EF e f +E. F. J. e f j +E. F. K. e f k +EFL e f l +EFM e f m +EFNB e f n b +EFPA e f p a +EFP e f p +EFQUEL e f q u e l +EFR e f r +EFRU e f r u +EFSA e f s a +eFS e f s +Efs e f's +EFSET e f s e t +EFTPOS e f t p o s +EFUA e f u a +Efu e f u +EFV e f v +EFVR e f v r +EFVS e f v s +E. F. W. e f w +EFX e f x +EFY e f y +E. G. A. e g a +EGA e g a +Egba e g b a +EGBA e g b a +EGBT e g b t +EGC e g c +EGD e g d +EGF e g f +egfl e g f l +EGFL e g f l +e.g. for example +e. g. for example +EGFR e g f r +EG&G e g and g +E. G. H. e g h +Egi e g i +EGL e g l +EGM e g m +E&GR e and g r +EGR e g r +EGREM e g r e m +EGSC e g s c +EGX e g x +egy e g y +EGY e g y +Egyl e g y l +EHAAT e h a a t +EHAs e h a's +EHC e h c +EHCI e h c i +EHD e h d +E. H. e h +E.H. e h +ehf e h f +EHF e h f +E. H. M. e h m +EHP e h p +EHRC e h r c +EHR e h r +EHS e h s +EHSH e h s h +Ehttp e h t t p +EIA e i a +EIC e i c +EiCs e i c's +EIC's e i c's +E. I. e i +E.I. e i +EIES e i e s +eIF e i f +EIF e i f +EIFF e i f f +EIF's e i f's +EIT e i t +EITI e i t i +Eitr e i t r +EITs e i t's +EIU e i u +E. J. A. e j a +EJB e j b +EJBs e j b's +EJC e j c +ej e j +E. J. e j +E.J. e j +EJ e j +E. J. H. e j h +E.J.H. e j h +EJH e j h +EJHL e j h l +EJMA e j m a +E. J. M. e j m +EJML e j m l +E. J. R. e j r +E.J.S. e j s +EJSM e j s m +EJV e j v +EKB e k b +EKBO e k b o +EKCO's e k c o's +EKD e k d +E. K. e k +E.K. e k +EKF e k f +EKG e k g +eki e k i +EKIZ e k i z +EKL e k l +EKLR e k l r +Ekma e k m a +EKNU e k n u +Ekow e k o w +eks e k s +Eks e k's +EKS e k s +E. K. U. e k u +EKU e k u +E. L. C. e l c +E. L. e l +E.L. e l +ELH e l h +ELTs e l t's +ELUs e l u's +ELV e l v +EMA e m a +EMAS e m a s +Embd e m b d +EMB e m b +eMC e m c +EMC e m c +EMD e m d +EMD's e m d's +EMDs e m d's +EMDS e m d s +Emea e m e a +EMEA e m e a +E&M e and m +E. M. e m +E.M. e m +EMF e m f +Emge e m g e +EMG e m g +EMLL e m l l +E.M.M. e m m +EMNLP e m n l p +EMP e m p +emra e m r a +EMR e m r +EMSC e m s c +E.M.S. e m s +EMTE e m t e +EMT e m t +EMV e m v +E&NA e and n a +Enas e n a's +ENAS e n a s +EnBW e n b w +ENC e n c +EncFS e n c f s +Encyc e n c y c +E.N.D. e n d +ENEC e n e c +E. N. e n +E.N. e n +Enes e n e's +engl e n g l +Engl e n g l +ENGM e n g m +Engr e n g r +Eni e n i +ENI e n i +enn e n n +Enn e n n +ENP e n p +ENR e n r +ens e n s +Ens e n's +ENS e n s +Enso e n s o +ENSO e n s o +ENTJs e n t j's +ENTP e n t p +ENTV e n t v +EOC e o c +EOC's e o c's +EoD e o d +EOD e o d +E. O. e o +E.O. e o +EOFFTV e o f f t v +EOG e o g +Eois e o i's +EOKA e o k a +eok e o k +Eole e o l e +Eol e o l +EOL e o l +Eolss e o l s's +Eom e o m +EOOW e o o w +eop e o p +eoptta e o p t t a +Eora e o r a +EOR e o r +Eorl e o r l +Eors e o r's +EOSHD e o s h d +EOTC e o t c +EOTDC e o t d c +EOU e o u +E.O.W. e o w +E&P e and p +Epe e p e +E. P. E. e p e +EPEG e p e g +ep e p +Ep e p +E. P. e p +E.P. e p +EP e p +EPF e p f +EPFL e p f l +EPG e p g +EPHAR e p h a r +EPHB e p h b +Eph e p h +EPH e p h +Ephs e p h's +EPIA e p i a +epi e p i +Epi e p i +EPI e p i +E.P.J. e p j +EPL e p l +EPLF e p l f +EPM e p m +EPOC e p o c +ePO e p o +EPO e p o +epos' e p o's +epos e p o s +Epos e p o's +EPRDF e p r d f +EPR e p r +EPRI e p r i +EPRO e p r o +EPSC e p s c +E.P.s e p's +EPT e p t +EPV e p v +EPZA e p z a +EPZ e p z +EQAO e q a o +eq e q +Eq e q +EQ e q +EQG e q g +EQN e q n +EQP e q p +EQS e q s +EQT e q t +E.R.A. e r a +ERAF e r a f +ERAU e r a u +ERCB e r c b +ERC e r c +ERCES e r c e s +ERCIM e r c i m +ERCS e r c s +E. R. D. e r d +E. R. e r +E.R. e r +ERF e r f +ERH e r h +ERJs e r j's +ERK e r k +ERP e r p +ERPM e r p m +ERPMRP e r p m r p +ERRC e r r c +E. R. R. e r r +E.R.R. e r r +ERSAR e r s a r +ERT e r t +Eru e r u +ERU e r u +ERUUF e r u u f +Eruv e r u v +Erv e r v +ERW e r w +ES3 e s three +Esa e s a +E.S.A. e s a +ESA e s a +Esam e s a m +Esa's e s a's +ESA's e s a's +Esat e s a t +ESB e s b +ESC e s c +ESCHL e s c h l +ESCs e s c's +Esd e s d +ESD e s d +ESDI e s d i +E.S.E. e s e +E. S. e s +E.S. e s +ESF e s f +ESFG e s f g +E.S.G. e s g +ESG e s g +ESGR e s g r +esi e s i +ESI e s i +ESL e s l +ESM e s m +ESMLC e s m l c +ESMT e s m t +ESNAGI e s n a g i +ESN e s n +ESPA e s p a +ESPC e s p c +ESPD e s p d +ESP e s p +espn e s p n +ESPN e s p n +ESPNhttp e s p n h t t p +ESPN's e s p n's +ESPNU e s p n u +ESPNW's e s p n w's +esq e s q +Esq e s q +Esq. esquire +Esraa e s r a a +Esra e s r a +ESRB e s r b +ESRC e s r c +Esref e s r e f +ESR e s r +ESRF e s r f +Esri e s r i +ESRI e s r i +ESSR e s s r +estd e s t d +Estd e s t d +EST e s t +ESTP e s t p +esu e s u +ESU e s u +ESU's e s u's +Esva e s v a +ESV e s v +ESWAT e s w a t +ESW e s w +Etad e t a d +ETA e t a +ETAF e t a f +ETA's e t a's +ETASU e t a s u +ETB e t b +ETBs e t b's +E.T.C. e t c +ETCO's e t c o's +ETCS e t c s +ETCSL e t c s l +ETD e t d +E. T. e t +E.T. e t +ETF e t f +ETFs e t f's +ETFS e t f s +Et'hem e t h e m +ETH e t h +Eti e t i +ETI e t i +ETIM e t i m +ETIM's e t i m's +etj e t j +ETJ e t j +ETL e t l +ETNZ e t n z +Eto e t o +ETO e t o +ETRM e t r m +ETRS e t r s +ETSEIB e t s e i b +ets e t s +Ets e t's +ET's e t's +ETs e t's +ETS e t s +ett e t t +Ett e t t +ETT e t t +Ettre e t t r e +Etts e t t's +ETU e t u +ETV e t v +Etz e t z +EUA e u a +EUB e u b +EUBs e u b's +EUBS e u b s +EUCAP e u c a p +EUCE e u c e +EUC e u c +EUCOM's e u c o m's +EUDEC e u d e c +EUD e u d +E. U. e u +E.U. e u +EU e u +eup e u p +EUP e u p +Eurwg e u r w g +EUSA e u s a +Euse e u s e +EU's e u's +EUS e u s +EUSEW e u s e w +Eusi e u s i +EUV e u v +EUX e u x +EVAAP e v a a p +EvaGT e v a g t +EVDC e v d c +EVD e v d +EVDO e v d o +E. V. e v +E.V. e v +EVEX e v e x +EVIT e v i t +evl e v l +EVL e v l +EVM e v m +Evna e v n a +EVN e v n +EVNS e v n s +evnt e v n t +E.V.O. e v o +E.V.O.L. e v o l +EVP e v p +EVR e v r +EVs e v's +EVS e v s +evv e v v +EVV e v v +Ewa's e w a's +EWD e w d +EWEB e w e b +E. W. e w +E.W. e w +EWF e w f +EWF's e w f's +Ewha e w h a +EWH e w h +E.W.K. e w k +EWLA e w l a +EWL e w l +EWO e w o +EWOs e w o's +EWP e w p +EW's e w's +EWS e w s +EWTN e w t n +Ewu e w u +EWU e w u +excl e x c l +exd e x d +E.X. e x +exhb e x h b +exh e x h +EXI e x i +EXIF e x i f +EXIN e x i n +EXL e x l +EXLM e x l m +E. X. M. e x m +Exptl e x p t l +EXR e x r +exsul e x s u l +EXT e x t +EYA e y a +EYCN e y c n +EYD e y d +E&Y e and y +E. Y. e y +EY e y +Eyk e y k +eyu e y u +Ezaa e z a a +ez e z +Ez e z +EZ e z +Ezh e z h +EZH e z h +EZO e z o +EZR e z r +EZTV e z t v +Ezu e z u +EZW e z w +Ezy e z y +Faaa f a a a +faa f a a +Faa f a a +FAA f a a +faama f a a m a +FAANP f a a n p +FabH f a b h +FABM f a b m +FabR f a b r +F.A.B.'s f a b's +F.A.C.E. f a c e +FACR f a c r +F.A.C.S. f a c s +FADD f a d d +Fadl f a d l +FADS f a d s +FAEM f a e m +fa f a +Fa f a +F. A. f a +F.A. f a +FA f a +Faf f a f +FAFG f a f g +Fafhrd f a f h r d +FAFL f a f l +F. A. G. f a g +F.A.H. f a h +fai f a i +Fai f a i +FAI f a i +FAIM f a i m +FAIP f a i p +FAIPs f a i p's +Faiq f a i q +fajn f a j n +FAK f a k +Fal f a l +FAL f a l +F.A.M.E. f a m e +F.A.M. f a m +FAQ f a q +FARC's f a r c's +FARDC f a r d c +F.A.R. f a r +farw f a r w +FASA f a s a +FASB f a s b +fasc f a s c +Fasc f a s c +fas f a s +Fas f a's +FAs f a's +F.A.S. f a s +FAS f a s +FASG f a s g +FasL f a s l +FASRE f a s r e +FATF f a t f +FAUP f a u p +FAVC f a v c +FBA f b a +FBD f b d +FBFA f b f a +F. B. f b +F.B. f b +FB f b +F.B.G.s f b g's +F.B.I. f b i +FBI f b i +FBI's f b i's +FBK f b k +FBL f b l +FBMW f b m w +FBN f b n +FBO f b o +FBOs f b o's +FBPA f b p a +FBR f b r +FBS f b s +FBSN f b s n +FBSU f b s u +FBT f b t +FCA f c a +fcb f c b +FCB f c b +FCBF f c b f +F.C.B.I. f c b i +FCBL f c b l +FCBs f c b's +F.C.C. f c c +FCC f c c +FCD f c d +F&C f and c +fc f c +F. C. f c +F.C. f c +FCF f c f +FCGR f c g r +F. C. H. f c h +F.C.H.L. f c h l +FCIAC f c i a c +FCIC f c i c +FCI f c i +FCIP f c i p +FCK f c k +FCL f c l +FCNM f c n m +FCNZ f c n z +FCO f c o +FCP f c p +F.C.P.R.E.M.I.X. f c p r e m i x +FCPS f c p s +FCRLA f c r l a +F.C.'s f c's +FC's f c's +FCS f c s +FCT f c t +FCU f c u +FCUL f c u l +FCV f c v +F. C. W. f c w +FCW f c w +FCW's f c w's +FDA f d a +FDCC f d c c +FD&C f d and c +FDCPA f d c p a +FDCPAs f d c p a's +FDD f d d +FDDI f d d i +FDE f d e +F. D. f d +F.D. f d +FDFL f d f l +FDGB f d g b +FDG f d g +FDICs f d i c's +FdI f d i +FDI f d i +FDIO f d i o +FDJ f d j +FDL f d l +FDLT f d l t +FDM f d m +FDMNES f d m n e s +FDNY f d n y +FDP f d p +FDP's f d p's +FDR f d r +FDU f d u +F. E. f e +F.E. f e +FEFK f e f k +FEFSI f e f s i +FEFY f e f y +FEG f e g +FEGs f e g's +FEP f e p +F.E.R.S. f e r s +Fes f e's +FES f e s +FFA f f a +FFAK f f a k +ffc f f c +FFC f f c +FFC's f f c's +FFDO f f d o +ff f f +F. F. f f +F.F. f f +FF f f +FFF f f f +FFG f f g +FFH f f h +FFI f f i +FFK f f k +FFL f f l +FFL's f f l's +FFME f f m e +ffm f f m +FFOM f f o m +FFP f f p +FFPIR f f p i r +FFR f f r +FFSA f f s a +FF's f f's +FFs f f's +FFS f f s +FFU f f u +FFV f f v +FFWD f f w d +FGCU f g c u +FGFA f g f a +FGF f g f +F. G. f g +F.G. f g +FGFR f g f r +FGID f g i d +fgk f g k +FGL f g l +FGM f g m +FGR f g r +FGs f g's +FGS f g s +FGU f g u +FGV f g v +F. H. A. f h a +FHA f h a +FHAR f h a r +F. H. C. f h c +FHFA f h f a +F. H. f h +F.H. f h +FHI's f h i's +FHL f h l +fhm f h m +FHM f h m +FHM's f h m's +FHQ f h q +FHSAA f h s a a +FHS f h s +FHWA f h w a +F. H. W. f h w +FIAC f i a c +fiadh f i a d h +fia f i a +FIA f i a +FIAPO f i a p o +FIA's f i a's +FIAS f i a s +FIBCO f i b c o +F. I. B. f i b +FiBL f i b l +FIBT f i b t +FIC f i c +Fi'd f i d +FIDH f i d h +F.I f +F&I f and i +F. I. f i +F.I. f i +FIFTA f i f t a +FIGC f i g c +Figl f i g l +FIH f i h +FIH's f i h's +fija f i j a +FIJL f i j l +FIK f i k +F.I.M. f i m +FIM f i m +FIMI f i m i +Fio f i o +FiO f i o +FIO f i o +F.I.R. f i r +F.I.T.T. f i t t +FIU f i u +FIVB f i v b +F. J. A. f j a +FJE f j e +F. J. f j +F.J. f j +F.J.G. f j g +FJK f j k +F. J. M. f j m +FKA f k a +FKBP f k b p +F. K. f k +F.K. f k +FKK f k k +FKL's f k l's +FKS f k s +FKY f k y +Fla f l a +FLB f l b +FLCL f l c l +FLCS f l c s +FLD f l d +FLDS f l d s +FLEC f l e c +FLE f l e +Fles f l e's +FLEST f l e s t +FLETC f l e t c +fl f l +F. L. f l +F.L. f l +FL f l +FLH f l h +fli f l i +FLI f l i +FLI's f l i's +FLL f l l +F.L.M. f l m +FLM f l m +FLNB f l n b +FLN f l n +FLP f l p +FLPP f l p p +FLQ f l q +FLRA f l r a +FLR f l r +FLRL f l r l +FLS f l s +FLSW f l s w +FLT f l t +FLTK f l t k +FLTR f l t r +FLTS f l t s +FLV f l v +FLW f l w +flyr f l y r +FMA f m a +FMARS f m a r s +FMC f m c +FMCG f m c g +FMCSA f m c s a +FMCS f m c s +FMF f m f +fm f m +F. M. f m +F.M. f m +FM f m +FMH f m h +FMHS f m h s +FMI f m i +FMJD f m j d +FMLN f m l n +FMM f m m +FMN f m n +FMNJ f m n j +FMNP f m n p +FMPA's f m p a's +FMP f m p +FMQB f m q b +FMR f m r +fMRI f m r i +FMRP f m r p +FMRU f m r u +FM's f m's +FMs f m's +FMS f m s +FMSF's f m s f's +FMSO f m s o +FMV f m v +FMW f m w +FMX f m x +FMY f m y +FNAC f n a c +F. N. A. f n a +FNA f n a +FNB f n b +FNC f n c +F. N. D. f n d +FNE f n e +F. N. f n +F.N. f n +FNIC f n i c +FNI f n i +FNLA f n l a +FNLC f n l c +FNLC's f n l c's +FNL f n l +FNO f n o +FNP f n p +FNPI f n p i +FNSEA f n s e a +FNU f n u d s a +FOF f o f +F. O. f o +FPA f p a +FPB f p b +FPC f p c +FPCP f p c p +F.P.D. f p d +FPDs f p d's +fp f p +F. P. f p +F.P. f p +FP f p +FPGA f p g a +FPGAs f p g a's +FPI f p i +FPJ f p j +FPK f p k +F. P. L. f p l +FPL f p l +FPMR f p m r +FPM&SA f p m and s a +FPO f p o +FPP f p p +FPPS f p p s +F.P.R. f p r +FPR f p r +FPSC f p s c +fps f p s +FPS f p s +FPSO f p s o +FPSs f p s's +FPTP f p t p +FPU f p u +FQAD f q a d +F.Q. f q +FRA f r a +FRA's f r a's +FRAXA f r a x a +FRB f r b +FRBNF f r b n f +FRCC f r c c +F. R. C. f r c +FRC f r c +FRCNA f r c n a +FRCP f r c p +FRCS f r c s +FRDF f r d f +F.R.D. f r d +fr f r +F. R. f r +F.R. f r +FR f r +FRFS f r f s +FRFT f r f t +FRG f r g +F.R.G.S. f r g s +F.R.H.S. f r h s +F.R.I.B.A. f r i b a +FRMN f r m n +FRMR f r m r +FRMS f r m s +F. R. P. f r p +FRP f r p +FRR f r r +F.R.S.A. f r s a +FRSA f r s a +FRSC f r s c +FRSE f r s e +frs f r s +F. R. S. f r s +F.R.S. f r s +FRS f r s +FRSL f r s l +Fru f r u +FRU f r u +FRV f r v +FRWT f r w t +Frwydr f r w y d r +FSA f s a +FSB f s b +FSC f s c +FSCS f s c s +FSE f s e +fsf f s f +FSF f s f +FSFH f s f h +FSFLA f s f l a +fs f s +F. s f's +Fs f's +F. S. f s +F.S. f s +FS f s +FSFS f s f s +FSG f s g +FSGS f s g s +FSH f s h +FSIC f s i c +FSI f s i +FSIL f s i l +FSK f s k +FSKN f s k n +FSLF f s l f +FSL f s l +FSLN f s l n +FSM f s m +FSM's f s m's +fsn f s n +FSN f s n +FSO f s o +FSP f s p +FSPS f s p s +FSR f s r +FSRU f s r u +FSSB f s s b +FSS f s s +FSSM f s s m +FSSRU f s s r u +FSSs f s s's +FST f s t +FSU f s u +FSU's f s u's +FSV f s v +FTA f t a +FTAs f t a's +F.T.B. f t b +FTC f t c +FTD f t d +FTDI f t d i +FTE f t e +FTF f t f +Ft. Fort +F. T. f t +F.T. f t +FTI f t i +FTII f t i i +FTJ f t j +FTK f t k +FTL f t l +FTM f t m +FTN f t n +FTO f t o +ftp f t p +FTP f t p +FTPI f t p i +FTPL f t p l +FTRA f t r a +FTR f t r +FtsA f t s a +FT's f t's +FTS f t s +FTT f t t +FTTH f t t h +FTUC f t u c +Ftuh f t u h +FTU's f t u's +FTV f t v +F. V. B. f v b +F. V. f v +F.V. f v +FVGM f v g m +FVHS f v h s +FVJC f v j c +FVL f v l +FVP f v p +FVW f v w +FVWM f v w m +FWAA f w a a +FWA f w a +FWCD f w c d +FWD f w d +FWE f w e +F&W f and w +F. W. f w +F.W. f w +FWH f w h +FWI f w i +FWMW f w m w +FWO f w o +fwr f w r +F.W.R. f w r +F. W. S. f w s +F.W.S. f w s +FWS f w s +FWX f w x +FXCM's f x c m's +FXE f x e +FXFL f x f l +fx f x +FX f x +FXI f x i +F.X.R. f x r +FX's f x's +FXS f x s +FXX f x x +FXX's f x x's +FXXT f x x t +Fyb f y b +FYCO f y c o +F.Y. f y +F.Y.I. f y i +FYI f y i +Fyn f y n +FYN f y n +FYP f y p +fyr f y r +F.Y.R. f y r +FYR f y r +FYT f y t +FYU f y u +Fyw f y w +FZCI f z c i +FZD f z d +F.Z. f z +FZG f z g +FZJD f z j d +FZKA f z k a +Gaac g a a c +GAAF g a a f +G.A.A. g a a +G. A. g a +G.A. g a +G.A.R. g a r +GASL g a s l +GATA g a t a +GATV g a t v +GATX g a t x +G. A. W. g a w +GBA g b a +GBAG g b a g +GBASE g b a s e +GBA's g b a's +GBB g b b +GBBI g b b i +GBCA g b c a +GBC g b c +GBCM g b c m +GBD g b d +Gbe g b e +GbE g b e +GBE g b e +G. B. g b +G.B. g b +GBGC g b g c +GBGM g b g m +GBH g b h +GBIB g b i b +GBIF g b i f +GBI g b i +GBK g b k +GBL g b l +GBMA g b m a +G. B. M. g b m +GB&NDR g b and n d r +GBP g b p +GBR g b r +GB's g b's +GBS g b s +GBV g b v +GBX g b x +GBYLA g b y l a +GCAC g c a c +GCAF g c a f +GCA g c a +G. C. B. g c b +GCB g c b +GCC g c c +GCCN g c c n +GCC's g c c's +GCCS g c c s +GCD g c d +G.C.E. g c e +GCEP g c e p +GCF g c f +G&C g and c +G. C. g c +G.C. g c +GCHQ g c h q +GCI g c i +GCIP g c i p +GCIT's g c i t's +GCIV g c i v +GCJ g c j +GCL g c l +GCMC g c m c +G.C.M. g c m +GCM g c m +GCMG g c m g +GCN g c n +GCNT g c n t +GCOEC g c o e c +GCOS g c o s +GCPD g c p d +GCPH g c p h +GCR g c r +GCRIO g c r i o +GCSE g c s e +GCSEs g c s e's +GC&SF g c and s f +GCS g c s +GCSI g c s i +GCU g c u +GCV g c v +GCVO g c v o +GCVS g c v s +GDA g d a +Gdal g d a l +GDAL g d a l +GDAP g d a p +GDB g d b +GDC g d c +GDD g d d +GDDM g d d m +GDDR g d d r +Gde g d e +G. D. F. g d f +GDF g d f +gd g d +G. D. g d +G.D. g d +GD g d +GDI g d i +G.D.L. g d l +GDN g d n +G.D.O.D. g d o d +Gdow g d o w +G.D.P. g d p +GDP g d p +GDPs g d p's +gdr g d r +GDR g d r +GDR's g d r's +GDS g d s +GDST g d s t +GDT g d t +GDV g d v +Gebr g e b r +Gebt g e b t +GEC g e c +Ged g e d +GED g e d +GEF g e f +GEGB g e g b +ge g e +G. E. g e +G.E. g e +GE g e +GEHA g e h a +GEHX g e h x +Gek g e k +G.E.M. g e m +Gen. general +G.E.O. g e o +G. E. P. g e p +geq g e q +Geu g e u +GEU g e u +Gev g e v +GeV g e v +GEV g e v +Gfa g f a +GFA g f a +GF&A g f and a +GFC g f c +GFD g f d +GFDL g f d l +GFE g f e +GFF g f f +G. F. g f +G.F. g f +GFM g f m +GFP g f p +G. F. R. g f r +GFR g f r +GFS g f s +GFSN g f s n +GFSR g f s r +GFT g f t +GFWC g f w c +GFW g f w +GFY g f y +G. G. g g +G.G. g g +G. G. J. g g j +GGL g g l +GGM g g m +GGMRC g g m r c +GGS g g s +G.H.A. g h a +GHB g h b +GHC g h c +GHD g h d +Ghe g h e +G. H. g h +G.H. g h +GHGs g h g's +Ghir g h i r +GHK g h k +GHMC g h m c +G.H.N. g h n +G.H.Q. g h q +GHQ g h q +ghra g h r a +GHR g h r +GH&SA g h and s a +GHSA g h s a +GHSFHA g h s f h a +GHS g h s +GHSR g h s r +GHSU g h s u +GHU g h u +G.H.W. g h w +GICL g i c l +Giei g i e i +GIFV g i f v +G. I. g i +G.I. g i +G.J.B. g j b +G. J. g j +G.J. g j +G. K. g k +G.K. g k +Gla g l a +GLA g l a +GLB g l b +GLBT g l b t +GLBTQ g l b t q +GLC g l c +G&L g and l +G. L. g l +G.L. g l +gli g l i +Gli g l i +GLI g l i +Glis g l i's +GLK g l k +GLL g l l +GLLI g l l i +GLM g l m +glnA g l n a +G.L.O.R.Y. g l o r y +Glos g l o's +GLRS g l r s +GLS g l s +GLT g l t +GLVC g l v c +GLX g l x +GMAC g m a c +gma g m a +GMA g m a +GMAM g m a m +GMA's g m a's +GMAT g m a t +GMB g m b +GmbH g m b h +GMBH g m b h +GMB's g m b's +GMCC g m c c +GMC g m c +GMCL g m c l +GMC's g m c's +GMDSS g m d s s +GME g m e +Gmel g m e l +GMES g m e s +GMFF g m f f +GMF g m f +GMG g m g +gm g m +G. M. g m +G.M. g m +GM g m +GMG's g m g's +gmin g m i n +GMM g m m +GMO g m o +GMOs g m o's +GMPC g m p c +GMPE g m p e +GMP g m p +GMPTE g m p t e +GMR g m r +GMRL g m r l +GM's g m's +GMs g m's +GMS g m s +gmt g m t +GMT g m t +GMTV g m t v +Gmul g m u l +GNAA g n a a +GNA g n a +gnb g n b +GNB g n b +GNCC g n c c +G. N. C. g n c +GNC g n c +G. N. g n +G.N. g n +GNL g n l +GNP&BR g n p and b r +GNPDA g n p d a +gnp g n p +GNP g n p +GNPs g n p's +GNRD g n r d +GNR g n r +GNR's g n r's +GNS g n s +GNSS g n s s +GNT g n t +G.O. g o +G.O.O.D. g o o d +GOPB g o p b +G.O.P. g o p +GOP g o p +GOPIO g o p i o +GOP's g o p's +GOQ g o q +GPA g p a +GPB g p b +G.P.C. g p c +GPC g p c +GPCR g p c r +GPCRs g p c r's +GPD g p d +GPdI g p d i +GPDSC g p d s c +GPF g p f +GPFS g p f s +G&P g and p +GPG g p g +gp g p +G. P. g p +G.P. g p +GP g p +GPGPU g p g p u +GPI g p i +GpIIb g p i i b +GPIO g p i o +GPL g p l +GPMGs g p m g's +G.P.O. g p o +GPO g p o +GPP g p p +GPRA g p r a +GPRF g p r f +GPR g p r +GPRs g p r's +GPRS g p r s +GPSC g p s c +GPS g p s +GPT g p t +GPU g p u +GPUs g p u's +GPX g p x +G. Q. g q +GQ g q +GQNYGK g q n y g k +GQ's g q's +GQT's g q t's +G.R.A.B. g r a b +grac g r a c +gra g r a +Gra g r a +GRA g r a +Grbac g r b a c +GRB g r b +GRBs g r b's +grc g r c +GRC g r c +GRCs g r c's +GRDA g r d a +GRDC g r d c +G.R.D. g r d +GRD g r d +gre g r e +Gre g r e +GRE g r e +GRF g r f +GRG g r g +gr g r +G. R. g r +G.R. g r +GR g r +GRK g r k +G.R.L. g r l +GRL g r l +GRMG g r m g +GRM g r m +GRO g r o +GRP g r p +grrl g r r l +G. R. S. g r s +GRS g r s +GRT g r t +GRU g r u +GRX g r x +Gryf g r y f +gry g r y +Gry g r y +GRY g r y +grz g r z +GSA g s a +GSAPP g s a p p +GSAS g s a s +GSB g s b +GSC g s c +GSDF g s d f +GSD g s d +G.S.E. g s e +GSE g s e +GSFC g s f c +GSF g s f +G&S g and s +GSG g s g +Gs g's +G. S. g s +G.S. g s +GS g s +GSHD g s h d +GSH g s h +GSI g s i +GSIs g s i's +GSIS g s i s +GSK g s k +GSL g s l +GSLV g s l v +GSMA g s m a +GSM g s m +GSNAS g s n a s +G. S. N. g s n +GSN g s n +GSNM g s n m +GSN's g s n's +GSO g s o +GSOM's g s o m's +GSOp g s o p +GSPC g s p c +G. S. P. g s p +GSP g s p +GSR g s r +GSSE g s s e +GSS g s s +GSSP g s s p +GSTA g s t a +G.S.T. g s t +GST g s t +GSU g s u +GSUSA g s u s a +GSVD g s v d +GSVS g s v s +GSX g s x +GTAA g t a a +GTA g t a +GTAs g t a's +GTB g t b +GTBK g t b k +GTBP g t b p +GTC g t c +GTCR g t c r +GTD g t d +GTEC g t e c +GTE g t e +GTF g t f +GTFT g t f t +GTFU g t f u +gt g t +G. T. g t +G.T. g t +GT g t +G. T. H. g t h +GTi g t i +GTI g t i +GTK g t k +GTL g t l +GTMO g t m o +G. T. O. g t o +GTO g t o +GTPBP g t p b p +GTP g t p +GTPR g t p r +GTRA g t r a +GTR g t r +GTRI g t r i +GTs g t's +GTS g t s +GTSV g t s v +GTT g t t +GTTP g t t p +GTU g t u +GTV g t v +GTW g t w +GTW's g t w's +GTX g t x +G. U. g u +G.U. g u +GUI's g u i's +GUIs g u i's +G.U.Y. g u y +GVA g v a +GVAV g v a v +GVBA g v b a +GVC g v c +G. V. g v +G.V. g v +GVHSS g v h s s +GVJZ g v j z +GVK g v k +GvpA g v p a +GVP g v p +GVRD g v r d +GVSU g v s u +GVT g v t +GVU g v u +GVVV g v v v +GVW g v w +GVWR g v w r +GWA g w a +GWB g w b +G. W. C. g w c +GWe g w e +Gwet g w e t +G.W.F. g w f +GWF g w f +gw g w +G. W. g w +G.W. g w +GW g w +G. W. H. g w h +GWHS g w h s +Gwi g w i +GWK g w k +G.W.L. g w l +GWO g w o +GWOT g w o t +gwr g w r +GWR g w r +GWR's g w r's +GW's g w's +G. W. S. g w s +GWS g w s +GWU g w u +GXG g x g +GX g x +GXP g x p +GYAN's g y a n's +GYC g y c +GYF g y f +Gy g y +G. Y. g y +GY g y +Gyn g y n +GYN g y n +Gyps g y p's +gyu g y u +Gyu g y u +GZA g z a +GZ g z +GZR g z r +HAFV h a f v +H. A. h a +H.A. h a +H&A h and a +HBA h b a +HBA's h b a's +HBBO h b b o +HBC h b c +HBCo h b c o +HBC's h b c's +HBCU h b c u +HBES h b e s +HBF h b f +HBGAs h b g a's +H. B. h b +H.B. h b +HB h b +HBI h b i +HBK h b k +HBLX h b l x +HBMP h b m p +HBO h b o +HBO's h b o's +HBPU h b p u +H&BR h and b r +HBR h b r +HBRN h b r n +HBRY h b r y +HB's h b's +HBS h b s +HBSS h b s s +HBU's h b u's +HBV h b v +HBW h b w +HCA h c a +HCAW h c a w +HCBE h c b e +HCB h c b +HCBS h c b s +HCCA h c c a +HCCB h c c b +HCCC h c c c +HCC h c c +HCDCH h c d c h +HCE h c e +HCEs h c e's +HCF h c f +H. C. G. h c g +HCGVN h c g v n +H. C. h c +H.C. h c +HC h c +HCH h c h +HCHS h c h s +HCI h c i +HCJB h c j b +hcl h c l +HCL h c l +HClO h c l o +HCMC's h c m c's +HCME h c m e +HCM h c m +HCN h c n +HCO h c o +HCoV h c o v +HCPA h c p a +H. C. P. h c p +HCP h c p +HCPs h c p's +HCR h c r +HCS h c s +HCT h c t +HCTV's h c t v's +HCUP h c u p +HCV h c v +Hcy h c y +HDAC h d a c +Hdad h d a d +hDAF h d a f +HDA h d a +HDB h d b +HDCAM h d c a m +HDC h d c +HDCP h d c p +HDD h d d +HDDs h d d's +HDFC h d f c +H.D.F. h d f +HDFM h d f m +H.D.G. h d g +HDG h d g +hd h d +H. D. h d +H.D. h d +HD h d +hdh h d h +HDi h d i +HDI h d i +HDLC h d l c +HDL h d l +HDLS h d l s +HDM h d m +HDMI h d m i +HDPE h d p e +HDP h d p +HDR h d r +HDRO h d r o +HDRR h d r r +H. E. h e +H.E. h e +hezb h e z b +Hezb h e z b +HFA h f a +HFB h f b +HFC h f c +HFCS h f c s +HFDF h f d f +HFF h f f +HFF's h f f's +H&F h and f +H. F. h f +H.F. h f +HFI h f i +H. F. J. h f j +HFNB h f n b +HFPA h f p a +HFP h f p +HFPPV h f p p v +HFRS h f r s +HFS h f s +HFTP h f t p +H.G.B. h g b +hgcA h g c a +HGC h g c +HGF h g f +hg h g +H. G. h g +H.G. h g +HG h g +HGH h g h +HGNC h g n c +HgO h g o +H. G. O. h g o +HGPIN h g p i n +HGPs h g p's +HGTV h g t v +HGTV's h g t v's +HgU h g u +HGVs h g v's +HGY h g y +HHA h h a +H. H. B. h h b +HHC h h c +HHD h h d +HHDP h h d p +HHG h h g +H&H h and h +HHHDB h h h d b +H. H. h h +H.H. h h +HHPC h h p c +HHSAA h h s a a +HHS h h s +H.H.T. h h t +H. I. h i +H.I. h i +H.I.V.E. h i v e +HIV h i v +HIY h i y +Hizb h i z b +hja h j a +HJC h j c +H. J. h j +H.J. h j +HJ h j +HJK h j k +H.J.M. h j m +HJN h j n +H. J. R. h j r +H.J.R. h j r +H. J. W. h j w +HJW h j w +HKAA h k a a +Hkam h k a m +HKB h k b +HKCEC h k c e c +HKCEE h k c e e +HKDF h k d f +HKD h k d +HKDSE h k d s e +HKFC h k f c +H&K h and k +H. K. h k +H.K. h k +HK h k +H. K. J. h k j +HKK h k k +HKMA h k m a +HKN h k n +H.K.P. h k p +HKP h k p +HKSAR h k s a r +HK's h k's +HKT h k t +HKTWU h k t w u +HKU h k u +Hla h l a +HLA h l a +HLBN h l b n +HLE h l e +HLH h l h +H. L. h l +H.L. h l +HLL h l l +HLLS h l l s +HLN h l n +HLN's h l n's +HLS h l s +HLT h l t +HLTV h l t v +Hluk h l u k +hlutr h l u t r +HLV h l v +Hly h l y +HMAC h m a c +HMA h m a +Hman h m a n +H.M.A.S. h m a s +HMAS h m a s +HMB h m b +HMC h m c +HMCS h m c s +HMDA h m d a +HMD h m d +HMDI h m d i +HMDP h m d p +HME h m e +HMEL h m e l +HMF h m f +HMGA h m g a +HMG h m g +HMGIC h m g i c +HMGN h m g n +H&M h and m +HMH h m h +hm h m +H. M. h m +H.M. h m +HM h m +HMI h m i +HMIS h m i s +HMMC h m m c +HMM h m m +HMMWV h m m w v +HMN h m n +HMNZS h m n z s +HMNZT h m n z t +HMO h m o +HMOs h m o's +HMPC h m p c +HMP h m p +HMRC h m r c +hMRE h m r e +H&M's h and m's +H. M. S. h m s +H.M.S. h m s +HMS h m s +HMSN h m s n +H.M.S.O. h m s o +HMSO h m s o +H. N. h n +H.N. h n +HNI h n i +HNK h n k +HNL h n l +HNLMS h n l m s +HNMT h n m t +HNN h n n +HNO h n o +HNS h n s +H. O. h o +H.O. h o +HPA h p a +H.P.B. h p b +hpc h p c +HPC h p c +HPCT h p c t +H. P. D. h p d +HPD h p d +HPE h p e +HPH h p h +hp h p +H. P. h p +H.P. h p +HP h p +HPI h p i +HPLC h p l c +HPL h p l +HPP h p p +HPRA h p r a +HPR h p r +HPRT h p r t +HP's h p's +HPS h p s +H. P. T. h p t +HPV h p v +HQAA h q a a +HQDA h q d a +H.Q. h q +HQ h q +HRG h r g +H&R h and r +H.R.H. h r h +HRH h r h +H. R. h r +H.R. h r +HR h r +HRK h r k +HRL h r l +HRM h r m +hroa h r o a +HRo h r o +HRO h r o +HRP h r p +HRSA h r s a +HRSA's h r s a's +HRSG h r s g +hrs h r s +HRs h r's +HRST h r s t +HRT h r t +HRV h r v +HRW h r w +HRX h r x +HSA h s a +HSAN h s a n +HSBC h s b c +HSB h s b +HSCA h s c a +HSC h s c +hscy h s c y +HSD h s d +HSDPA h s d p a +Hsee h s e e +HSE h s e +HSE's h s e's +HSF h s f +HSG h s g +HSHB h s h b +H.S.H. h s h +hs h s +Hs h's +H. S. h s +H.S. h s +HS h s +HSHS h s h s +hsi h s i +HSJ h s j +HSK h s k +HSL h s l +hSlo h s l o +H.S.M. h s m +HSM h s m +HSMR h s m r +HSN h s n +HSNY's h s n y's +HSPA h s p a +HSPA's h s p a's +HSPG h s p g +HSPH h s p h +HSP h s p +HSP's h s p's +HSPs h s p's +HSRC h s r c +HSR h s r +HSSCCT h s s c c t +HSSF h s s f +HSS h s s +HST h s t +HST's h s t's +HSTs h s t's +HSTS h s t s +hsv h s v +HSV h s v +HSX h s x +HSY h s y +HSYK's h s y k's +HSZ h s z +HTA h t a +hTAS h t a s +HTAs h t a's +htc h t c +HTC h t c +HTC's h t c's +HTGL h t g l +HTHC h t h c +ht h t +H. T. h t +H.T. h t +HT h t +HTIB h t i b +hti h t i +Htin h t i n +HTK h t k +HTLA h t l a +HTLV h t l v +HTML h t m l +HTM's h t m's +Htoo h t o o +HTPB h t p b +HTPC h t p c +HTP h t p +HtrA h t r a +HTR h t r +HTRs h t r's +HTS h t s +http h t t p +HTTP h t t p +https h t t p s +HTTPS h t t p s +HTV h t v +HTV's h t v's +Htwa h t w a +HTW h t w +HTX h t x +H. U. h u +HVC h v c +HVCs h v c's +hvcv h v c v +HVDC h v d c +hvem h v e m +Hvem h v e m +HVF h v f +HVHS h v h s +H. V. h v +H.V. h v +H.W.A. h w a +H. W. F. h w f +H. W. h w +H.W. h w +H.W.L. h w l +HWL h w l +HWM h w m +HWRT h w r t +HWT h w t +HWV h w v +hwy h w y +Hwy h w y +hxt h x t +H. Y. h y +H.Y. h y +H. Z. h z +HZ&PC h z and p c +iaaf i a a f +IAAF i a a f +IAAF's i a a f's +IAA i a a +IAAPA i a a p a +IAAP i a a p +IaaS i a a s +IAAS i a a s +IABSE i a b s e +Iaca i a c a +IACC i a c c +IACD i a c d +IAC i a c +Iacob i a c o b +IACR i a c r +IAC's i a c's +IACTs i a c t's +Iacub i a c u b +IACUC i a c u c +IACW i a c w +IAD i a d +IAD's i a d's +IAEA i a e a +IAEG i a e g +I.A.E. i a e +IAE i a e +IAF i a f +IAF's i a f's +IAG i a g +IAG's i a g's +iagt i a g t +IAHP i a h p +IAHRC i a h r c +I. A. i a +I.A. i a +I.A.M. i a m +IAO i a o +Iapa i a p a +IAPHC i a p h c +IAP i a p +iar i a r +Iar i a r +IAR i a r +IAST i a s t +IATA i a t a +I.A.T. i a t +IAT i a t +IATR i a t r +IAUC i a u c +IAUCs i a u c's +IAU i a u +IAV i a v +IAWTV i a w t v +Iax i a x +IAX i a x +Ibac i b a c +Ibaes i b a e's +IBA's i b a's +IBB i b b +IBC i b c +IBCs i b c's +IBCT i b c t +IBDA'A i b d a a +Ibda i b d a +ibdal i b d a l +IBDB i b d b +IBDE i b d e +IBD i b d +IBE i b e +IBF i b f +IBGE i b g e +IBH i b h +I. B. i b +I.B. i b +IBJJF i b j j f +IBL i b l +IBLT i b l t +IBMA's i b m a's +IBMC i b m c +Ibm i b m +I.B.M. i b m +IBM i b m +IBM's i b m's +IBMXF i b m x f +Ibne i b n e +ibn i b n +IBO i b o +IBPA i b p a +IBP i b p +Ibra i b r a +IBRA i b r a +IBRC i b r c +IBR i b r +IBRO i b r o +Ibsa i b s a +IBSA i b s a +IBSF i b s f +Ibs i b's +IBS i b s +IBSL i b s l +IBTC i b t c +iBT i b t +IBTS i b t s +Ibu i b u +IBU i b u +IBV i b v +IBWFF i b w f f +iby i b y +ICBA i c b a +ICBC i c b c +ICB i c b +ICBL i c b l +ICBM i c b m +ICBMs i c b m's +ICBP i c b p +ICBS i c b s +ICBTT i c b t t +ICCC i c c c +ICCC's i c c c's +ICCEC i c c e c +ICCF i c c f +Icche i c c h e +I.C.C. i c c +ICC i c c +ICCID i c c i d +ICCI i c c i +ICCR i c c r +ICC's i c c's +ICCS i c c s +ICCW i c c w +ICDC i c d c +ICD i c d +ICDS i c d s +I.C.E. i c e +ICFF i c f f +icf i c f +ICF i c f +ICFR i c f r +ICG i c g +ICHC i c h c +I&C i and c +ic i c +Ic i c +I. C. i c +I.C. i c +IC i c +Ici i c i +ICI i c i +ICJ i c j +ICL i c l +ICL's i c l's +ICMA i c m a +ICME i c m e +I.C.M. i c m +ICM i c m +iCN i c n +ICN i c n +Ico i c o +ICO i c o +ICOP i c o p +ICPC i c p c +ICPD i c p d +ICP i c p +ICP's i c p's +ICQ i c q +ICRC i c r c +ICRF i c r f +ICR i c r +IcRn i c r n +ICRP i c r p +ICRS i c r s +ICRW i c r w +ICSA i c s a +ICSB i c s b +ICSC i c s c +ICSE i c s e +ICSG i c s g +ics i c s +ICs i c's +ICS i c s +ICSS i c s s +ICSSR i c s s r +ICTA i c t a +ICT i c t +ICTM i c t m +ICTP i c t p +ICTSF i c t s f +ICTs i c t's +ICTS i c t s +ICTU i c t u +ICTUR i c t u r +ICTVdB i c t v d b +ICTV's i c t v's +ICTY i c t y +ICU i c u +ICUN i c u n +ICU's i c u's +ICv i c v +ICV i c v +ICVS i c v s +ICWA i c w a +ICWC i c w c +ICW i c w +ICW's i c w's +iCyt i c y t +ICZ i c z +ICZN i c z n +IDAF i d a f +IDBF i d b f +IDB i d b +IDBI i d b i +IDC i d c +Iddaa i d d a a +Idd i d d +ID'd i d d +IDEF i d e f +IDE i d e +IDFA i d f a +IDFG i d f g +IDF i d f +IDFL i d f l +IDF's i d f's +IDG i d g +I. D. H. i d h +IDH i d h +id i d +I. D. i d +I.D. i d +ID i d +Idi i d i +Idir i d i r +IDIS i d i s +IDLHs i d l h's +IDL i d l +Idm i d m +IDM i d m +IDMR i d m r +IDMS i d m s +IDNA i d n a +IDN i d n +IDNR i d n r +IDNR's i d n r's +IDPs i d p's +IDRC i d r c +IDRO i d r o +IDSA i d s a +IDSA's i d s a's +IDSF i d s f +ids i d s +ID's i d's +IDs i d's +IDV i d v +IDW i d w +IDWO i d w o +IDW's i d w's +IDx i d x +IDX i d x +IEA i e a +I.E.M. i e m +IEM i e m +I.E.S. i e s +IES i e s +i.e. that is +Ifa i f a +IFA i f a +IFBB i f b b +IFBC i f b c +I.F.B. i f b +IFCC i f c c +ifc i f c +IFC i f c +IFC's i f c's +IfF i f f +I. F. F. i f f +IFF i f f +IFFI i f f i +IFFK i f f k +Ifft i f f t +IFHA i f h a +IFHP i f h p +ifi i f i +Ifi i f i +IFI i f i +IFIP i f i p +IFIs i f i's +IFK i f k +IFLA i f l a +IFL i f l +IfM i f m +IFM i f m +IFN i f n +Ifni i f n i +IFNs i f n's +IFO i f o +Ifop i f o p +IFP i f p +IFPI i f p i +IFPRI i f p r i +IFRC i f r c +IFR i f r +IFRS i f r s +IFRSs i f r s's +IFSB i f s b +IFSC i f s c +IFSTA i f s t a +IFTA i f t a +IFTF i f t f +IFT i f t +Iga i g a +IgA i g a +IGA i g a +IGA's i g a's +IGBT i g b t +IGC i g c +IGCP i g c p +IGCSE i g c s e +IGD i g d +IGF i g f +IGFI i g f i +IGFs i g f's +IGIB i g i b +'ig i g +Ig i g +I.G. i g +IG i g +igi i g i +IGI i g i +IgM i g m +IGM i g m +IGMP i g m p +ign i g n +Ign i g n +IGN i g n +IHF i h f +IHGS i h g s +IHH i h h +Ih i h +I. H. i h +IH i h +ihi i h i +IHI i h i +IHL i h l +IHL's i h l's +IHMC i h m c +IHME i h m e +ihm i h m +ihn i h n +Ihn i h n +IHOM i h o m +Ihor i h o r +IHPK i h p k +IHRC i h r c +IHSAA i h s a a +IHSA i h s a +IHS i h s +Iht i h t +IIA i i a +IIb i i b +IIB i i b +IIc i i c +IIDC i i d c +IID i i d +IIDI i i d i +IIDS i i d s +IIDX i i d x +IIED i i e d +IIFA i i f a +IIF i i f +IIfx i i f x +Iiga i i g a +IIga i i g a +IIG i i g +IIgs i i g's +IIGs i i g's +IIGS i i g s +IIGS's i i g s's +IIHF i i h f +IIHR i i h r +IIHS i i h s +IIIA i i i a +IIIb i i i b +IIIB i i i b +IIIBy i i i b y +IIIc i i i c +IIId i i i d +IIIF i i i f +I. I. i i +I.I. i i +ija i j a +Ija i j a +IJA i j a +IJF i j f +IJGL i j g l +IJHCD i j h c d +IJHL i j h l +ij i j +I. J. i j +IJ i j +IJN i j n +Ijok i j o k +IJSB i j s b +IJSE i j s e +IJSEM i j s e m +IJS i j s +Ijui i j u i +IJVM i j v m +IKBN i k b n +IKCO i k c o +IKF i k f +Ikh i k h +I.K. i k +IKK i k k +IKL i k l +IKr i k r +Iksa i k s a +IKTA i k t a +Iku i k u +Ilbe i l b e +ILB i l b +ILC i l c +ILCS i l c s +ILD i l d +ILDIS i l d i s +ILFC i l f c +ILF i l f +IL&FS i l and f s +ili i l i +Ili i l i +ILI i l i +I. L. i l +I.L. i l +Ilm i l m +ILM i l m +ILM's i l m's +ILN i l n +ILP i l p +ILR i l r +ILRS i l r s +ilu i l u +Ilu i l u +ILU i l u +IMbd i m b d +IMBD i m b d +IMB i m b +IMCB i m c b +IMCI i m c i +imc i m c +IMC i m c +imdb i m d b +IMDb. i m d b +IMDb i m d b +IMDB i m d b +IMD i m d +IMD's i m d's +ime i m e +IME i m e +IMF i m f +IMG i m g +I&M i and m +IMI i m i +I. M. i m +I.M. i m +IMIS i m i s +IMK i m k +Iml i m l +IMMEX i m m e x +Imm i m m +Imms' i m m's +imoa i m o a +IMO's i m o's +IMPA i m p a +IMPDH i m p d h +impr i m p r +Impr i m p r +IMPSA i m p s a +IMR i m r +IMSLP i m s l p +iMSNs i m s n's +Imst i m s t +IMTA i m t a +IMT i m t +IMTV i m t v +IMUK i m u k +IMU's i m u's +IMWF i m w f +INAC i n a c +Inba i n b a +INBA i n b a +INBF i n b f +inb i n b +incl i n c l +INDC i n d c +ind i n d +Ind i n d +IND i n d +INFJ i n f j +I.N.F.O. i n f o +Infs i n f's +ING i n g +INGR i n g r +inHg i n h g +INH i n h +INI i n i +I. N. i n +I.N. i n +InlB i n l b +INL i n l +INM i n m +INPD i n p d +INPE i n p e +InP i n p +INP i n p +INPP i n p p +Inre i n r e +I.N.R.I. i n r i +INR i n r +INSB i n s b +INSDC i n s d c +INSD i n s d +Inspx i n s p x +INSS i n s s +INTF i n t f +Ints i n t's +INTs i n t's +Intu i n t u +INVA i n v a +inv i n v +INXS i n x s +INXS's i n x s's +Ioba i o b a +IOCG i o c g +IOC i o c +IOCP i o c p +IODP i o d p +IODs i o d's +IOE i o e +IOFB i o f b +IOF i o f +io. i o +io i o +.I.o. i o +Io. i o +Io i o +.IO i o +I/O i o +IO i o +Ioka i o k a +Iok i o k +IOL i o l +IoM i o m +IOM i o m +IOPI i o p i +IOP i o p +IOPs i o p's +IORG i o r g +IOR i o r +IORM i o r m +IOSA i o s a +IOSB i o s b +iOS i o s +Io's i o's +I/Os i o's +I.O.'s i o's +IOS i o s +IOSR i o s r +Iosu i o s u +IoT i o t +IOT i o t +I.O.U. i o u +IOU i o u +IoW i o w +IOW i o w +IPA i p a +IPA's i p a's +IPB i p b +IPCC i p c c +IPCC's i p c c's +IPC i p c +IPCRI i p c r i +IPCs i p c's +IPDA i p d a +IPDI i p d i +IPD i p d +IPE i p e +IPFA i p f a +IPFI i p f i +IPF i p f +IPFP i p f p +IPFW i p f w +IPG i p g +IPH i p h +ip i p +Ip i p +I.P. i p +IP i p +IPL i p l +IPMG i p m g +IPM i p m +IPNI i p n i +IPN i p n +Ipo i p o +I.P.O. i p o +IPO i p o +IPP i p p +IPPL i p p l +IPPs i p p's +IPPT i p p t +Ippu i p p u +iPPV i p p v +IPR i p r +iPSC i p s c +IPSC i p s c +iPSCs i p s c's +ipse i p s e +IPSF i p s f +iPS i p s +IPs i p's +I.P.S. i p s +IPS i p s +IPTA i p t a +IPTC i p t c +IPTG i p t g +IPT i p t +IPTL i p t l +IPTV i p t v +IPU i p u +IPUM i p u m +IPv i p v +IPWA i p w a +IPW i p w +IPX i p x +iQ i q +I.Q. i q +IQ i q +IRAC i r a c +I.R.A. i r a +IRB i r b +IRBM i r b m +IRCA i r c a +IRC i r c +IRCM i r c m +IRCTC i r c t c +IRCT i r c t +IRD i r d +IRDS i r d s +IREC i r e c +I.R.E. i r e +IRE i r e +IRFCA i r f c a +IRFC i r f c +IRF i r f +IRFU i r f u +IRGC i r g c +Irgm i r g m +IRHG i r h g +IRHT i r h t +IRIB i r i b +Irig i r i g +IRI i r i +Iril i r i l +IRIN i r i n +i'r i r +ir i r +Ir i r +I. R. i r +I.R. i r +IR i r +IRIX i r i x +Irla i r l a +IRL i r l +IRM i r m +Irmis i r m i's +IRM's i r m's +IRNE i r n e +IRN i r n +iro i r o +Iro i r o +IRO i r o +IRP i r p +IRP's i r p's +IRPS i r p s +IRQ i r q +Irra i r r a +IRRI i r r i +IRR i r r +IRs i r's +I.R.S. i r s +IRS i r s +IRSN i r s n +I.R.T. i r t +IRT i r t +iru i r u +Iru i r u +ISA i s a +ISAv i s a v +Isba i s b a +ISBA i s b a +ISBER i s b e r +ISBHF i s b h f +ISB i s b +isbn i s b n +ISBN i s b n +ISBNs i s b n's +ISBO i s b o +ISCB i s c b +ISCE i s c e +isCf i s c f +ISCI i s c i +iSC i s c +Isc i s c +ISC i s c +ISCM i s c m +ISCM's i s c m's +ISCP i s c p +ISDA i s d a +Isd i s d +ISD i s d +isdn i s d n +ISDN i s d n +ISDS i s d s +ISDT i s d t +ISFI i s f i +ISF i s f +Isgec i s g e c +ISG i s g +ISHC i s h c +ishq i s h q +Ishq i s h q +ISHS i s h s +I. S. i s +I.S. i s +ISKB i s k b +ISK i s k +Isl i s l +ISL i s l +ISMF i s m f +ISMIR i s m i r +ISMNs i s m n's +ISN i s n +ISNTUC i s n t u c +ISOGG i s o g g +ISO i s o +ISO's i s o's +I.S.P.C.K. i s p c k +ISPF i s p f +ISPI i s p i +ISP i s p +ISPL i s p l +ISPR i s p r +ISPs i s p's +Isra'il i s r a i l +Isra i s r a +ISRA i s r a +ISR i s r +ISRN i s r n +ISRO i s r o +ISRP i s r p +ISSCC i s s c c +ISSF i s s f +iss i s s +Iss i s's +ISS i s s +ISSN i s s n +ISSNs i s s n's +ISSSSP i s s s s p +ISSUU i s s u u +ISTAF i s t a f +ISTD i s t d +Isu i s u +ISU i s u +ISU's i s u's +ISWC i s w c +ISWI i s w i +ISW i s w +I.T.A. i t a +ITA i t a +ITB i t b +ITC i t c +Ite i t e +ITE i t e +ITF i t f +ITG i t g +ITHF i t h f +I.T.H.M. i t h m +ITHs i t h's +iti i t i +Iti i t i +ITI i t i +I.T. i t +itk i t k +Itk i t k +ITK i t k +ITM i t m +ITMO i t m o +ITN i t n +ITNNs i t n n's +ITN's i t n's +Itoi i t o i +ITPF i t p f +iTP i t p +ITP i t p +ITRD i t r d +Itse i t s e +ITSF i t s f +ITTF i t t f +itt i t t +ITT i t t +ITUC i t u c +Itu i t u +ITU i t u +ITV i t v +ITV's i t v's +ITWF i t w f +ITW i t w +Itz i t z +IUB i u b +I.U.C.N.B. i u c n b +Iucn i u c n +IUCN i u c n +IUCN's i u c n's +IUD i u d +IUDs i u d's +IU i u +IUT i u t +IUU i u u +Iuz i u z +IVAA i v a a +IVAW i v a w +IVB i v b +I. V. i v +I.V. i v +IVL i v l +IVLP i v l p +IVM's i v m's +IVP i v p +IVPP i v p p +IVPTC i v p t c +IVRCL i v r c l +IVs i v's +Ivu i v u +IVWA i v w a +IWA i w a +Iwas i w a's +Iwaz i w a z +IWBs i w b's +IWC i w c +IWD i w d +IWF i w f +IWFL i w f l +IWF's i w f's +IWFS i w f s +IWG i w g +IWGP i w g p +iwi i w i +iw i w +I. W. i w +IW i w +IWL i w l +IWMF i w m f +IWM i w m +IWPR i w p r +iwrg i w r g +IWRG i w r g +IWR i w r +IWRS i w r s +IWSA i w s a +iwspy i w s p y +Iwuh i w u h +IWU's i w u's +I.W.W. i w w +IWW i w w +IXA i x a +IXb i x b +IXBs i x b's +IXC i x c +IXCs i x c's +IXe i x e +IX i x +iXL i x l +IXO i x o +ixtle i x t l e +IXV i x v +Iya i y a +IYA i y a +Iyar i y a r +IYC i y c +iyem i y e m +IYHF i y h f +Iyi i y i +iy i y +I. Y. i y +I.Y. i y +IY i y +J.A.A. j a a +J. A. B. j a b +Jaf j a f +J.A.G. j a g +JAG j a g +JAIR j a i r +J. A. j a +J.A.K.E. j a k e +J. A. N. j a n +J.A.P. j a p +J.A.S. j a s +JBA j b a +JBBA j b b a +JBB j b b +JBC j b c +JBG j b g +J&B j and b +jb j b +J. B. j b +J.B. j b +JB j b +J.B.L.D. j b l d +JBL j b l +J.B.M. j b m +JBM j b m +JB's j b's +J. B. S. j b s +J.B.S. j b s +JBT j b t +JBZ j b z +JCA j c a +JCB j c b +JCC j c c +JCF j c f +JCFS j c f s +J.C.G. j c g +JCG j c g +JCGM j c g m +JCIB j c i b +JCI j c i +jcis j c i s +jc j c +J. C. j c +J.C. j c +JC j c +JCJC j c j c +J. C. L. j c l +JCPA j c p a +J. C. P. j c p +JCP j c p +JCPV j c p v +JCRB j c r b +JCR j c r +JCRs j c r's +J.C.S. j c s +JCS j c s +JCSR j c s r +JCVI j c v i +J. C. W. j c w +JCW j c w +J. D. A. j d a +JDA j d a +JDBC j d b c +JDC j d c +JdeBP j d e b p +J.D.E. j d e +JDE j d e +JDENET j d e n e t +J.D.F. j d f +JDF j d f +jdi j d i +JDI j d i +jd j d +J. D. j d +J.D. j d +JD j d +jdk j d k +JDK j d k +J.D.M. j d m +JDM j d m +JDO j d o +JDRF j d r f +J&D's j and d's +J.D.s j d's +JDT j d t +JDU j d u +JDV j d v +J. E. A. j e a +J.E.A.L. j e a l +J.E.B. j e b +J. E. C. j e c +J. E. E. j e e +J. E. j e +J.E. j e +J.F.G. j f g +JFH j f h +jf j f +J. F. j f +J.F. j f +JF j f +JFK j f k +JFN j f n +JFQA j f q a +JFS j f s +J. G. A. j g a +J.G.B. j g b +JGI j g i +J. G. j g +J.G. j g +J. G. L. j g l +JGLS j g l s +J. G. M. j g m +J.G.M. j g m +JGO j g o +JGP j g p +JGR j g r +JGR's j g r's +JGSDF j g s d f +JGTC j g t c +JGU j g u +J.H.B. j h b +JHB j h b +J. H. C. j h c +JHC j h c +JHED j h e d +J.H.F. j h f +J.H.H. j h h +jh j h +J. H. j h +J.H. j h +JH j h +JHL j h l +J. H. M. j h m +J.H.R. j h r +JHs j h's +JHS j h s +JHSVs j h s v's +JHud j h u d +JHU j h u +J. I. j i +J.I. j i +J.J.A. j j a +JJB j j b +J. J. C. j j c +J.J.G. j j g +J&J j and j +J. J. j j +J.J. j j +JJ j j +JJK j j k +J. J. L. j j l +J.J.M. j j m +J.J.N. j j n +JJN j j n +Jka j k a +JKA j k a +JKC j k c +jkd j k d +JKH j k h +J&K j and k +J.K. J. j k j +J. K. j k +J.K. j k +JK j k +JKNPP j k n p p +J. K. S. j k s +JKT j k t +jkx j k x +JKX j k x +JLA j l a +J.L.B. j l b +JLB j l b +JLC j l c +J. L. E. j l e +J.L.E. j l e +JLG j l g +JLI j l i +J. L. j l +J.L. j l +J.L.L. j l l +JLM j l m +JLO's j l o's +JLPGA j l p g a +JLP j l p +JLS j l s +JLT j l t +JMA j m a +JMBG j m b g +J.M.C. j m c +JMC j m c +J. M. D. j m d +JMD j m d +J. M. E. j m e +JMI j m i +J. M. J. j m j +J.M.J. j m j +J. M. j m +J.M. j m +JMKR j m k r +J. M. L. j m l +JML j m l +JMM j m m +JMP j m p +JMS j m s +JMU j m u +JMV j m v +J. M. W. j m w +J.M.W. j m w +JMX j m x +JNA j n a +jnb j n b +JNF j n f +JNG j n g +JNI j n i +J. N. j n +J.N. j n +JN j n +JNK j n k +J. N. L. j n l +Jno j n o +JNPT j n p t +jnr j n r +JNR j n r +JNS's j n s's +JOJ j o j +J. O. j o +J.O. j o +J.P.D.B. j p d b +J. P. F. j p f +JPF j p f +jpg j p g +JPG j p g +JPH j p h +J&P j and p +jp j p +J.P. j p +JP j p +J.P.R. j p r +J.Q. j q +JQ j q +J.R.A. j r a +JRA j r a +J.R.C. j r c +JRC j r c +J.R.E. j r e +JRG j r g +JRHU j r h u +J. R. I. j r i +J&R j and r +J. R. j r +J.R. j r +JR j r +JRJ's j r j's +Jr. junior +jr. junior +Jr junior +jr junior +JRPG j r p g +J. R. P. j r p +J. R. R. j r r +J.R.R. j r r +J.R.'s j r's +JRSP j r s p +J. R. T. j r t +JRU j r u +JRX j r x +JSA j s a +JSAP j s a p +JSA's j s a's +jsb j s b +JSCA j s c a +JSC j s c +J.S.D. j s d +JSD j s d +JSE j s e +JSF j s f +JSH j s h +Js j's +J. S. j s +J.S. j s +JS j s +J.S.K. j s k +J.S.L. j s l +JSL j s l +J. S. M. j s m +JSM j s m +JSNN j s n n +JSOC j s o c +JSO j s o +JSP j s p +JSPS j s p s +JSR j s r +JSS j s s +JSW j s w +JSW's j s w's +JSX j s x +JTA j t a +J.T.B. j t b +JTB j t b +JTC j t c +JTF j t f +JTG j t g +JTI j t i +J&T j and t +J. T. j t +J.T. j t +JT j t +JT&KW j t and k w +JTL j t l +JTO j t o +JTR j t r +JTRO j t r o +JTS j t s +JTTF j t t f +JTT j t t +JTV j t v +J. U. j u +J.U.L.I.A. j u l i a +J.V.B. j v b +JVB j v b +JVC j v c +J.V.E. j v e +JVG j v g +JVH j v h +J&V j and v +J. V. j v +J.V. j v +JV j v +JVM j v m +JVNW j v n w +JVP j v p +JVRA j v r a +JVS j v s +Jwa j w a +J. W. A. j w a +J. W. B. j w b +JWB j w b +J.W.C. U.S. j w c u s +JWHA j w h a +J.W.J. j w j +J. W. j w +J.W. j w +JWM j w m +J. W. P. j w p +JWP j w p +JWRC j w r c +J. W. S. j w s +J.W.S. j w s +JWS j w s +JWST j w s t +JWT j w t +jx j x +JX j x +J.X.W.P. j x w p +JYJ j y j +JYJ's j y j's +J. Y. j y +J.Y. j y +JY j y +Jym j y m +Jymn j y m n +JYP j y p +J. Y. S. j y s +J. Z. j z +J.Z. j z +JZ j z +K.A.A. k a a +K. A. k a +K.A. k a +K.A.N. k a n +K. A. R. k a r +K.A.S. k a s +KBBI k b b i +KBB k b b +KBCI k b c i +KBC k b c +KBCO k b c o +KBD k b d +KBE k b e +KBFC k b f c +KBFL k b f l +KBFR k b f r +KBGD k b g d +KBG k b g +KBIG k b i g +kbi k b i +K&B k and b +K. B. k b +K.B. k b +KBKR k b k r +KBKS k b k s +KBKW k b k w +KBL k b l +KBMT k b m t +KBND k b n d +KBNZ k b n z +KBO k b o +KBRC k b r c +KBR k b r +KBS k b s +KBSN k b s n +KCB k c b +KCBS k c b s +KCCC k c c c +KCCI k c c i +KCC k c c +KCCO k c c o +KCDC k c d c +K. C. D. k c d +KCD k c d +KCED k c e d +KCET k c e t +KCG k c g +KCHS k c h s +K.C.I.E. k c i e +K. C. k c +K.C. k c +KCK k c k +KCKM k c k m +KCLU k c l u +K.C.M.G. k c m g +KCMG k c m g +KCNA k c n a +KCNH k c n h +KCNJ k c n j +KCN k c n +KCNN k c n n +KCNQ k c n q +KCNV k c n v +KCPD k c p d +KCPI k c p i +KCPM k c p m +KCPQ k c p q +KCPQ's k c p q's +KCQL k c q l +KCRA k c r a +KCRC k c r c +KCRH k c r h +KCR k c r +KCRW k c r w +KCSD k c s d +KCSG k c s g +K.C.S.I. k c s i +KCSI k c s i +K.C.'s k c's +KCS k c s +KCSP k c s p +KCTD k c t d +KCT k c t +KCTL k c t l +KCTV k c t v +KCUB k c u b +KCU k c u +KCVO k c v o +KCYX k c y x +kDa k d a +KDA k d a +kdal k d a l +KDAL k d a l +KDB k d b +KDCE k d c e +KDCO k d c o +KDDG k d d g +KDDI k d d i +KDD k d d +KDE k d e +KDFW k d f w +KDFX k d f x +K. D. G. k d g +KDGS k d g s +KDHS k d h s +KDKA k d k a +K&D k and d +kd k d +K. D. k d +K.D. k d +KD k d +KDP k d p +KDPM k d p m +KDR k d r +K. D.'s k d's +K.D.'s k d's +KDS k d s +ke k e +Ke k e +K. E. k e +K.E. k e +KE k e +KFAC k f a c +KFAED k f a e d +KFAI k f a i +KFA k f a +KFBK k f b k +KFBT k f b t +KFCC k f c c +KFCD k f c d +kfc k f c +K.F.C. k f c +KFC k f c +KFC's k f c's +KFD k f d +KFDM k f d m +KFEM k f e m +KFFL k f f l +KFH k f h +KFI k f i +KFJB k f j b +K.F. k f +KFLX k f l x +KFMB's k f m b's +KFMJ k f m j +K. F. M. k f m +K.F.M. k f m +KFOR k f o r +KFQC k f q c +KFQ k f q +KFRC k f r c +KFRC's k f r c's +KFRD k f r d +KFRG k f r g +KFRH k f r h +KFRO k f r o +KFSD k f s d +KFSN k f s n +KFST k f s t +KFT k f t +KFTL k f t l +KFUM's k f u m's +KFVD k f v d +KFV k f v +KFWB k f w b +KFXO k f x o +KFYO's k f y o's +KGA k g a +K.G.B. k g b +KGB k g b +KGC k g c +KGPH k g p h +KGP k g p +KGRI k g r i +kgr k g r +KGRK k g r k +KGS k g s +kgt k g t +KGT k g t +KGTV k g t v +KgU k g u +KGU k g u +KGUN k g u n +kgv k g v +K.G.V. k g v +KGWB k g w b +KGW k g w +KHHZ k h h z +KHI k h i +Khizr k h i z r +KHJ k h j +K. H. k h +K.H. k h +KH k h +KHK k h k +KHKV k h k v +Khlav k h l a v +KHL k h l +KHL's k h l's +Khmu k h m u +KHMY k h m y +KHOJ k h o j +KHOW k h o w +KHQA k h q a +KHQ k h q +Khri k h r i +Khru k h r u +KHRW k h r w +KHSAA k h s a a +KHS k h s +KHTL k h t l +KHTZ k h t z +KHUI k h u i +KHUT k h u t +KHWI k h w i +K.I.D. k i d +K. I. k i +K.I. k i +K.I.M. k i m +KIP k i p +K.I.S.S. k i s s +KJCE k j c e +KJCT k j c t +KJEF k j e f +Kjer k j e r +KJHK k j h k +KJI k j i +KJIN k j i n +kj k j +K. J. k j +K.J. k j +KJ k j +KJKP k j k p +KJMM k j m m +KJNE k j n e +KJNW k j n w +Kjop k j o p +KJo's k j o's +KJQY k j q y +KJV k j v +KJZI k j z i +KKBL k k b l +KKBQ k k b q +KKCR k k c r +KKDA k k d a +K.K.E. k k e +KKE k k e +K. K. k k +K.K. k k +KKMC k k m c +KKMI k k m i +KKM k k m +KKMX k k m x +KKRG k k r g +KKR k k r +KKSF k k s f +KK's k k's +KKSY k k s y +KKT k k t +KLA k l a +KLBK k l b k +KLBM k l b m +KLCI k l c i +KLC k l c +KLDS k l d s +Kle k l e +KLE k l e +KLF k l f +KLGH k l g h +KLH k l h +KLK k l k +kl k l +K. L. k l +K.L. k l +KL k l +KLKS k l k s +KLLP k l l p +KLLV k l l v +KLM k l m +KLN's k l n's +KLQQ k l q q +KLQT k l q t +KLRY k l r y +KLSC k l s c +KLSD k l s d +KLS k l s +KLSX k l s x +KLT k l t +KLX k l x +KMB k m b +KMC k m c +KMD k m d +KMGO k m g o +KMGT k m g t +KMIH k m i h +KMIR k m i r +KMITL k m i t l +KMJ k m j +K&M k and m +KMK k m k +km k m +K. M. k m +K.M. k m +KM k m +KMLA k m l a +KML k m l +KMME k m m e +KMML k m m l +KMRL k m r l +KMS k m s +KMSP k m s p +KMT k m t +KMTP k m t p +KMTV k m t v +KMVA k m v a +KMXW k m x w +KMZ k m z +KNFM k n f m +KNHP k n h p +K&N k and n +kn k n +K. N. k n +K.N. k n +KN k n +KNKT k n k t +KNLA k n l a +KNMI k n m i +KNNV k n n v +KNPN k n p n +KNPR k n p r +KNPU k n p u +KNRE k n r e +KNRS k n r s +KNS k n s +KNSM k n s m +KNST k n s t +KNTH k n t h +K.N.T. k n t +KNTN k n t n +KNTU k n t u +KNVA's k n v a's +KNVB k n v b +KNVN k n v n +KNX k n x +KNXT k n x t +K.O.D. k o d +k'o k o +K. O. k o +K.O. k o +KOTC k o t c +KOTD k o t d +K.O.T. k o t +KOWL k o w l +K. P. A. C. k p a c +KPAC k p a c +KPA k p a +KPAN k p a n +KPCC k p c c +KPC k p c +KPD k p d +KPE k p e +KPIF k p i f +KPI k p i +KPIs k p i's +K&P k and p +K. P. k p +K.P. k p +KP k p +KPLC k p l c +KPL k p l +KPLZ k p l z +KPMG k p m g +KPM k p m +KPNA k p n a +KPNB k p n b +KPO k p o +Kppen k p p e n +KPPV k p p v +KPS k p s +kptm k p t m +KPTV k p t v +KPU k p u +KPVD k p v d +KPVX k p v x +KPWR k p w r +KPXE k p x e +KQA k q a +KQED k q e d +KQHN k q h n +KQKK k q k k +KQKS k q k s +KQLT k q l t +KQLZ k q l z +KQMO k q m o +KQV k q v +KQXR k q x r +KRBM k r b m +krc k r c +KRCR k r c r +KRCs k r c's +KRF k r f +KRH k r h +K.R.I.T. k r i t +K.R.I.T.'s k r i t's +Krka k r k a +KRKC k r k c +KRK k r k +kr k r +K. R. k r +K.R. k r +KR k r +KRMT k r m t +Krne k r n e +KRNS k r n s +KRNV k r n v +KROQ k r o q +kroz k r o z +Kroz k r o z +KRPS k r p s +KRRT k r r t +K. R. S. k r s +KRS k r s +KRT k r t +Kru k r u +KRU k r u +KRWC k r w c +KRX k r x +K.R.Y. k r y +KRZA k r z a +ksa k s a +KSA k s a +K. S. B. k s b +KSBW k s b w +KSBY k s b y +KSCB k s c b +KSC k s c +KSCO k s c o +KSC's k s c's +KSCS k s c s +KSDB k s d b +KSDO k s d o +KSEB k s e b +KSEG k s e g +KSE k s e +KSEQ k s e q +kset k s e t +KSET k s e t +Kseur k s e u r +Ksevt k s e v t +KSFO k s f o +KSG k s g +KSHB k s h b +KSI's k s i's +KSJR k s j r +KSKJ k s k j +KSKK k s k k +K.S.K. k s k +KSK k s k +ks k s +K. S. k s +K.S. k s +KS k s +KSLC k s l c +KSL k s l +KSLM k s l m +KSLU k s l u +KSMB k s m b +KSNR k s n r +KSNV k s n v +KSP k s p +KSPS k s p s +KSTE k s t e +KSTH k s t h +KST k s t +KSTP k s t p +KSU k s u +K.S.V. k s v +KSWD k s w d +KSWT k s w t +KSWW k s w w +KSYL k s y l +KSZR k s z r +Ktav k t a v +KTAV k t a v +KTBS k t b s +KTCC k t c c +KTC k t c +KTCZ k t c z +KTDA k t d a +KTDD k t d d +KTE k t e +KTH k t h +KT&K k t and k +kt k t +K. T. k t +K.T. k t +KT k t +KTKZ k t k z +KTLA k t l a +K. T. M. k t m +KTM k t m +KTN k t n +KTOK k t o k +KTO k t o +KTRE k t r e +KTRH k t r h +KTRK k t r k +KTR k t r +KT's k t's +KTSM k t s m +KTT k t t +KTTN k t t n +KTTV k t t v +KTU k t u +KTUM k t u m +KTVA k t v a +KTVK k t v k +KTVS k t v s +KTVT k t v t +KTVU k t v u +KTVZ k t v z +KTWD's k t w d's +KTWO k t w o +KTX k t x +KTXL k t x l +Kuaa k u a a +Kud k u d +KUFM k u f m +KUGB k u g b +kuih k u i h +K. U. k u +K.U. k u +Kutb k u t b +KUTV k u t v +KUUU k u u u +KUWL k u w l +KUYI k u y i +KVAB k v a b +KVBC k v b c +KVB k v b +K.V.G.K. k v g k +KVHV k v h v +Kvik k v i k +KVI k v i +Kvit k v i t +KVIT k v i t +KVK k v k +K. V. k v +K.V. k v +KVLO k v l o +KVLY k v l y +KVM k v m +KVMRT k v m r t +KVMX k v m x +KVNG k v n g +KVN k v n +KVP k v p +KVTV k v t v +KVVS's k v v s's +KVVV k v v v +Kvyat k v y a t +KVZ k v z +kwa k w a +KWBM k w b m +KWC k w c +KWD k w d +KWEI k w e i +kwe k w e +KWG k w g +KWJJ k w j j +KWKH k w k h +KWKW k w k w +KWMR k w m r +KWMT k w m t +KWNA k w n a +KWNK k w n k +KWOA k w o a +KWP k w p +KWPT k w p t +KWQC k w q c +KWRE k w r e +KWRU k w r u +KWSB k w s b +KWSX k w s x +KWU k w u +KWWL's k w w l's +KXAN k x a n +KXAS k x a s +KXI k x i +KXJB k x j b +KXKS k x k s +KX k x +KXLX k x l x +KXLY k x l y +KXMC k x m c +KXNA k x n a +KXOL's k x o l's +KXTA k x t a +KXTN k x t n +KXTX k x t x +KYAY k y a y +KYBE k y b e +KYEZ k y e z +KYNM k y n m +Kyse k y s e +KYSL k y s l +KYTC k y t c +kyt k y t +Kyt k y t +KYW k y w +kz k z +K. Z. k z +KZ k z +KZLZ k z l z +KZMP k z m p +KZMU k z m u +KZQX k z q x +KZZQ k z z q +L. A. G. l a g +L. A. l a +L.A.'s l a's +Lay's l a y's +L.B.A. l b a +LBA l b a +LBCC l b c c +LBC l b c +LBC's l b c's +LBE l b e +LBF l b f +LBi l b i +LBJ l b j +LBJ's l b j's +L. B. l b +L.B. l b +LB l b +LBL l b l +LBM l b m +LBN l b n +LBNL l b n l +LBP l b p +LBPs l b p's +LBR l b r +LBS l b s +LBV l b v +LBW l b w +LCAC l c a c +LCA l c a +LCAs l c a's +LCAS l c a s +L.C.B. l c b +LCBL l c b l +LCBM l c b m +lcc l c c +LCC l c c +LCCN l c c n +LCCs l c c's +LCCS l c c s +LCDB l c d b +LCD l c d +LCDP l c d p +LC&DR l c and d r +LCDR l c d r +LCDs l c d's +L.C.E. l c e +LCFC l c f c +LCF l c f +LCH l c h +LCI l c i +LCIs l c i's +L. C. l c +L.C. l c +LC l c +LCL l c l +LCMC l c m c +LCM l c m +LCMR l c m r +LCMV l c m v +lcn l c n +LCO l c o +LCP l c p +LCQ l c q +LCR l c r +LCSB l c s b +LCSC l c s c +LC's l c's +LCS l c s +LCSs l c s's +LCST l c s t +LCT l c t +LCTs l c t's +LCU l c u +LCV l c v +LCVP l c v p +LCVPs l c v p's +LCZ l c z +Lda l d a +LDA l d a +LDAP l d a p +LDB l d b +LDBV l d b v +L.D.C. l d c +LDC l d c +LDCM l d c m +LDCs l d c's +LDD l d d +LDDP l d d p +LDF l d f +LDH l d h +L.D.I. l d i +LDK l d k +L. D. l d +L.D. l d +LDL l d l +LDLR l d l r +LDM l d m +LDMR l d m r +LDN l d n +LDP l d p +LDS l d s +LDU l d u +LDV l d v +LEB l e b +L. E. l e +L.E. l e +LFA l f a +L.F.C. l f c +LFC l f c +L.F.D. l f d +LFE l f e +LFFCs l f f c's +LFF l f f +LFG l f g +LFHC l f h c +LFH l f h +L. F. l f +L.F. l f +LFL l f l +LFM l f m +LFOC l f o c +LFO l f o +LFOM l f o m +LFPB l f p b +LFP l f p +LFQD l f q d +LFRJ l f r j +LFR l f r +LFs l f's +LFS l f s +LFSRs l f s r's +LFTR l f t r +LFXA l f x a +LGA l g a +LGB l g b +LGBTI l g b t i +LGBT l g b t +LGBTQ l g b t q +LGC l g c +LGD l g d +LGI l g i +L. G. l g +L.G. l g +LG l g +LGMB l g m b +LGMs l g m's +LGN l g n +LGPL l g p l +LGPZ l g p z +LG&RDD l g and r d d +LGR l g r +LGs l g's +LGS l g s +LGTB l g t b +LGT's l g t's +LGU l g u +LGUs l g u's +LGV l g v +LGVs l g v's +Lha l h a +LHB l h b +L. H. C. l h c +LHC l h c +LHD l h d +LHFP l h f p +LHICE l h i c e +LHI l h i +L. H. l h +L.H. l h +LHMC l h m c +L.H.M. l h m +LHO l h o +L. H. P. l h p +LHP l h p +LHS l h s +LHW l h w +LHX l h x +LHY l h y +L. I. l i +L.I. l i +LJBL l j b l +Lje l j e +LJJ l j j +L. J. K. l j k +L. J. l j +L.J. l j +L.J.V. l j v +LKB l k b +LKG l k g +LKK l k k +L. K. l k +LK l k +LKL l k l +LKML l k m l +LKPR l k p r +LKRN l k r n +lks l k s +LKS l k s +LKTI l k t i +Llapi l l a p i +LLAW l l a w +L.L.B. l l b +LLB l l b +LLCC l l c c +L.L.C. l l c +LLC l l c +LLD l l d +LL.D l l d +lle l l e +LLE l l e +LLHS l l h s +lli l l i +LLI l l i +L&L l and l +LLLE l l l e +L. L. l l +L.L. l l +LL l l +LLL l l l +LLLP l l l p +LLM l l m +LLMNR l l m n r +LLPFX l l p f x +llp l l p +LLP l l p +LLPX l l p x +LLRW l l r w +LLT l l t +LLTV l l t v +llu l l u +LLVM l l v m +LLWS l l w s +LMA l m a +LMC l m c +LMCT l m c t +LMDB l m d b +LMDC l m d c +LMD l m d +LME l m e +LMFAO's l m f a o's +LMFF l m f f +LMG l m g +LMGs l m g's +LMGTE l m g t e +LMH l m h +LMHS l m h s +lm l m +L. M. l m +L.M. l m +LM l m +LMM l m m +LMP l m p +LMPs l m p's +LMQ l m q +LMQs l m q's +L.M.S. l m s +LMS l m s +LMT l m t +LMTP l m t p +LMU l m u +LMWH l m w h +LMX l m x +LnAIB l n a i b +LNA l n a +LNAV l n a v +LNBF l n b f +LNB l n b +L.N.C. l n c +LNCS l n c s +L.N.E.R. l n e r +LNER l n e r +LNFS l n f s +LNG l n g +LNH l n h +L&N l and n +L. N. l n +L.N. l n +L.O.C.'s l o c's +Lokk l o k k +l'OL l o l +L.O.L. l o l +L. O. l o +L.O. l o +L.O.V.E. l o v e +L.O.V. l o v +LOXL l o x l +LPA l p a +L.P.A.M. l p a m +LPAM l p a m +LPARs l p a r's +LPAVS l p a v s +LPBG l p b g +L.P.B. l p b +LPB l p b +LPC l p c +LPCM l p c m +LPDA l p d a +LPDR l p d r +LPE l p e +LPF l p f +LPFM l p f m +LPGA l p g a +LPG l p g +L.P.H. l p h +LPH l p h +LPI l p i +LPLA l p l a +L&P l and p +LPL l p l +lp l p +L. P. l p +L.P. l p +LP l p +L&PM l and p m +LPMN l p m n +LPMud l p m u d +LPMUD l p m u d +LPN l p n +LPO l p o +LPRP l p r p +LPSC l p s c +LP's l p's +LPs l p's +LPS l p s +LPSN l p s n +LPTB l p t b +LPThe l p t h e +LPTV l p t v +LQ l q +LRAD l r a d +LRA l r a +LRCD l r c d +LRC l r c +L.R.C.P. l r c p +LRCP l r c p +LRDG l r d g +LRG l r g +lr l r +L. R. l r +L.R. l r +LR l r +LRO l r o +L.R.P.C. l r p c +LRP l r p +LRPPRC l r p p r c +LRRI l r r i +LRRP l r r p +LRS l r s +LRTA l r t a +LRT l r t +LRTR l r t r +LRTs l r t's +LRVs l r v's +LRY l r y +LSAC l s a c +LSA l s a +LSBC l s b c +LSB l s b +LSC l s c +LSCS l s c s +LSDHH l s d h h +L.S.D. l s d +LSD l s d +LSDP l s d p +LSDs l s d's +LSE l s e +LSF l s f +LSG l s g +LSH l s h +LSi l s i +LSI l s i +LSJ l s j +LSK l s k +L.S.L. l s l +ls l s +Ls l's +L. S. l s +L.S. l s +LSM l s m +LSO l s o +LSP l s p +LSPN l s p n +LSQC l s q c +LSRI l s r i +L. S. R. l s r +LSS l s s +LSSP l s s p +LSSR l s s r +LST l s t +LSTM l s t m +LSTs l s t's +LSUA l s u a +LSU l s u +LSU's l s u's +LSVCCs l s v c c's +LSV l s v +LSWR l s w r +LTAF l t a f +L. T. B. l t b +LTB l t b +LTCF l t c f +LTCI l t c i +L. T. C. l t c +L.T.C. l t c +LTC l t c +Ltda l t d a +LTDA l t d a +Ltd. limited +LTE l t e +LTFA l t f a +LTHS l t h s +LTI l t i +LTK l t k +L&T l and t +Lt. lieutenant +LTL l t l +lt l t +L. T. l t +L.T. l t +L.T.M. l t m +LTM l t m +LTMPS l t m p s +LTN l t n +ltoh l t o h +LTP l t p +LTRPC l t r p c +LTTE l t t e +LTTR l t t r +LTU l t u +LTV l t v +L. U. l u +lv l v +L. V. l v +L.V. l v +LV l v +LVMH l v m h +LWDB l w d b +LWD l w d +LWDS l w d s +LWE l w e +LWF l w f +LWH l w h +L. W. l w +L.W. l w +L.Y. l y +L&YR l and y r +M.A.D.E. m a d e +M.A.D. m a d +M.A.J. m a j +M.A.K. m a k +M. A. m a +M.A. m a +M&A m and a +M. A. O. m a o +M.A.O. m a o +M.A.R. m a r +M.A.S.K. m a s k +M.A.S. m a s +M.A.S.S. m a s s +M.B.A. m b a +M.B.B.S. m b b s +MBBS m b b s +MBC m b c +MBC's m b c's +MBDA m b d a +MBD m b d +M.B. D.P.M. m b d p m +M.B.E. m b e +MBE m b e +Mbewu m b e w u +MBF m b f +MBFW m b f w +MBGN m b g n +MBH m b h +mbi m b i +MBI m b i +M. B. J. m b j +MBJ m b j +MBK m b k +MBL m b l +M. B. m b +M.B. m b +MBM m b m +MBNA m b n a +mBo m b o +MBO m b o +MBPJ m b p j +MBP m b p +Mbre m b r e +mbr m b r +MBR m b r +MBSE m b s e +MBSI m b s i +MBS m b s +MBTI m b t i +MBT m b t +MBU's m b u's +MBX m b x +MCAF m c a f +MCAL m c a l +M.C.A. m c a +MCA m c a +MCA's m c a's +MCAS m c a s +MCB m c b +M. C. C. m c c +M.C.C. m c c +MCC m c c +MCC's m c c's +MCCs m c c's +MCCU m c c u +MCDA m c d a +MCDC m c d c +MCD m c d +MCDM m c d m +MCE m c e +MCFA m c f a +M. C. F. m c f +MCF m c f +MCFM m c f m +MCGJCW m c g j c w +MCG m c g +MCI m c i +MCJ m c j +MCLA m c l a +MCL m c l +M. C. m c +M.C. m c +MCMC m c m c +M. C. M. m c m +MCM m c m +MCN m c n +MCO m c o +MCOT's m c o t's +MCPC m c p c +MCPI m c p i +MCP m c p +MCPON m c p o n +MCPP m c p p +MCQ m c q +MCRD m c r d +MCR m c r +MCRP m c r p +MCSBA m c s b a +MCs m c's +M.C. S. m c s +MCS m c s +MCST m c s t +MCTFS m c t f s +MCT m c t +MCTU m c t u +MCTV m c t v +MCU m c u +MCV m c v +MCVTS m c v t s +MCW m c w +mcyG m c y g +MCYO m c y o +MCZ m c z +MDAA m d a a +MDAH m d a h +MDA m d a +mdb m d b +MDB m d b +MDC m d c +MDC's m d c's +MDCs m d c's +MDCT m d c t +mdDA m d d a +MD&DI m d and d i +Mde m d e +MDE m d e +MDF m d f +MDG m d g +MDGs m d g's +MDH m d h +MDHUs m d h u's +mDia m d i a +MDIB m d i b +MDic m d i c +MDI m d i +MDJT's m d j t's +MDK m d k +mdla m d l a +MDL m d l +MDMA m d m a +md m d +M. D. m d +M.D. m d +MD m d +MDM m d m +MDNA m d n a +M.D.N. m d n +MDPD's m d p d's +MDP m d p +MDPPP m d p p p +MDPS m d p s +MDPV m d p v +MDQ m d q +MDRC m d r c +mdr m d r +MDR m d r +MDT m d t +MDV m d v +MDX m d x +M. E. m e +M.E. m e +M.E.N. m e n +M. E. P. m e p +M.F.A. m f a +MFA m f a +MFB m f b +MFCC m f c c +MFC m f c +MFCS m f c s +MFD m f d +MFDs m f d's +MFe m f e +MFF m f f +MFG m f g +MFI m f i +MFJ m f j +MFK m f k +M. F. m f +M.F. m f +MFMF m f m f +MFM m f m +MFN m f n +MFNW m f n w +MFP m f p +MFR m f r +MFS m f s +MFS's m f s's +MFTBC m f t b c +MFT m f t +MFTs m f t's +Mgadla m g a d l a +Mgal m g a l +mga m g a +Mga m g a +MGA m g a +MGB m g b +Mgbo m g b o +MGCCC m g c c c +MGC m g c +MGE m g e +MGen m g e n +MGG m g g +MGH m g h +MGIMO m g i m o +MGIT m g i t +MGJH m g j h +MGK m g k +M.G.L. m g l +M&G m and g +M. G. m g +M.G. m g +M. G. M. m g m +MGM m g m +MGM's m g m's +MGMT m g m t +M&GN m and g n +MGN m g n +MGO m g o +MGP m g p +M&GR's m and g r's +Mha m h a +M.H.A. m h a +MHA m h a +MHB's m h b's +MHC m h c +MHCs m h c's +MHD m h d +MHI m h i +MHK m h k +MH&L m h and l +MHL m h l +M&H m and h +M. H. m h +M.H. m h +Mhor m h o r +Mhow m h o w +MHPL m h p l +M.H.P. m h p +MHP m h p +MHP's m h p's +MHRA m h r a +M. H. R. m h r +MHSA m h s a +MHSC m h s c +MHS m h s +MHT m h t +M.I.A. m i a +M.I.A.'s m i a's +M.I.C.M. m i c m +M.I.H. m i h +M&I m and i +M. I. m i +M.I. m i +M.I.N. m i n +M.I.R.V. m i r v +M.I.S. m i s +M.I.T. m i t +MJAHL's m j a h l's +M.J.A. m j a +MJA m j a +MJBHA m j b h a +MJB m j b +MJC m j c +M.J.F. m j f +MJF m j f +MJG m j g +MJHL m j h l +MJHL's m j h l's +M&J m and j +M. J. m j +M.J. m j +M. J. Y. m j y +MKBHD m k b h d +MKB m k b +MKDE m k d e +MKE m k e +MKFM m k f m +MKG m k g +MKK m k k +MKMF m k m f +M. K. m k +M.K. m k +MKNG m k n g +MKP m k p +MKRN m k r n +MKs m k's +MKS m k s +MKT m k t +MKTO m k t o +MKTV m k t v +MKZ's m k z's +M.L.A. m l a +MLA m l a +MLANA m l a n a +MLAs m l a's +MLB m l b +MLBPA m l b p a +MLCA m l c a +mlc m l c +M.L.C. m l c +MLC m l c +MLCs m l c's +MLD m l d +MLE m l e +MLF m l f +mlg m l g +MLG m l g +MLH m l h +MLIA m l i a +mli m l i +MLI m l i +MLIM m l i m +MLK m l k +MLK's m l k's +Mlle m l l e +MLL m l l +MLLT m l l t +M&L m and l +M. L. m l +M.L. m l +MLM m l m +MLND m l n d +M.L.N. m l n +MLN m l n +MLPH m l p h +MLP m l p +MLPs m l p's +M. L. R. m l r +MLR m l r +MLSE m l s e +M. L. S. m l s +M.L.S. m l s +MLS m l s +MLT m l t +MLW m l w +MLWS m l w s +MM&A m m and a +M. M. J. m m j +MMK m m k +MML m m l +M&M m and m +M. M. m m +M.M. m m +MMSA m m s a +M&M's m and m's +M. M. S. m m s +MMS m m s +MMST m m s t +MMTB's m m t b's +MMTS m m t s +MMTV m m t v +MMWR m m w r +MMX m m x +MNA m n a +MNBA m n b a +MNC m n c +MNCPPC m n c p p c +MNCs m n c's +MNDM m n d m +MNDNR m n d n r +MNDO m n d o +MNDR m n d r +M&NF m and n f +MNF m n f +MNG m n g +MNI m n i +MNLA m n l a +MNLA's m n l a's +MNLF m n l f +MNM m n m +M. N. m n +M.N. m n +MNNA m n n a +MNN m n n +MNP m n p +MNPP m n p p +MNRG m n r g +MNSD m n s d +MNS m n s +MNTC m n t c +MNT m n t +MNZM m n z m +MNZ m n z +MOBKL m o b k l +MOBK m o b k +M.O.B. m o b +M.O.D. m o d +M.O.D.O.K. m o d o k +M.O.G.U.E.R.A.'s m o g u e r a's +M. O. H. m o h +M. O. m o +M.O. m o +M.O.P. m o p +M.O.R. m o r +M.O.T. m o t +MPAA m p a a +MpA m p a +MPA m p a +MPAs m p a's +MPB m p b +MPBN m p b n +MPBu m p b u +MPCA m p c a +MPCI m p c i +MPC m p c +MPD m p d +MPE m p e +MPF m p f +MPG m p g +M.P.H. m p h +MPH m p h +MP&I m p and i +MPi m p i +MPI m p i +MPIO m p i o +MPKAB m p k a b +MPLAD m p l a d +MPLA m p l a +MPL m p l +MPLMs m p l m's +MPLR m p l r +MPLS m p l s +M&P m and p +MPM m p m +M. P. m p +M.P. m p +MPO m p o +MPPJ m p p j +M.P.P. m p p +MPP m p p +MPPSC m p p s c +MPPs m p p's +MPQC m p q c +MPR m p r +MPRO m p r o +MPRP m p r p +MPRS m p r s +MPSE m p s e +MPSF m p s f +M.P.'s m p's +MP's m p's +MPs m p's +MPS m p s +MPThe m p t h e +MPT m p t +Mpu m p u +MPU m p u +MPV m p v +MPW m p w +MPZ m p z +MQM m q m +M. Q. m q +Mra m r a +MRA m r a +M. R. B. m r b +MRB m r b +MRBs m r b's +MRCB m r c b +MRCK m r c k +MRCM m r c m +MRC m r c +M.R.C.S. m r c s +MRCS m r c s +MRDC m r d c +MRD m r d +mre m r e +MRF m r f +MR&LE m r and l e +MRL m r l +Mr. mister +MRM m r m +M. R. m r +M.R. m r +mRNA m r n a +mRNAs m r n a's +MRPGM m r p g m +MRP m r p +MRPS m r p s +MRR m r r +MRSA m r s a +M.R.S.C. m r s c +Mrs. misses +MRSM m r s m +MRT m r t +MRTs m r t's +MRTS m r t s +MRTT m r t t +MRU m r u +MSAA m s a a +MSAC m s a c +MSAD m s a d +MSA m s a +M.S.A.S. m s a s +MSAW m s a w +MSB m s b +MSBO m s b o +MSBSD m s b s d +MSCC m s c c +M.S.C. m s c +MSC m s c +msd m s d +MSD m s d +MSDN m s d n +MSDS m s d s +Mse m s e +MSFC m s f c +M.S.F. m s f +MSF m s f +MSFT m s f t +MSG m s g +MSI m s i +MSJ m s j +MSK m s k +MSL m s l +M&S m and s +Ms. miss +MSML m s m l +M.S.M. m s m +MSM m s m +M. S. m s +M.S. m s +MSNBC m s n b c +MSNBC's m s n b c's +MSN m s n +MSOB m s o b +MSOM m s o m +M.S.P. m s p +MSP m s p +MSPs m s p's +MSR m s r +MSRP m s r p +MSRTC m s r t c +MSSH m s s h +MSSK m s s k +MSSMLP m s s m l p +MSSM m s s m +MSS m s s +MSTA m s t a +mst m s t +MST m s t +MSTS m s t s +MSU's m s u's +MSVCC m s v c c +MSV m s v +MSVU m s v u +MSX m s x +MSY m s y +MSZP m s z p +MTAC m t a c +Mta m t a +M.T.A. m t a +MTA m t a +MTA's m t a's +MTAs m t a's +MTBE m t b e +MTBI m t b i +MTB m t b +MTC m t c +MTDB m t d b +M.T.D. m t d +MTD m t d +MTD's m t d's +MTG m t g +MTHFR m t h f r +MTHL m t h l +MTIC m t i c +MTI m t i +MTJ m t j +MTK m t k +MTKO m t k o +M&T m and t +MTM m t m +MTMR m t m r +M. T. m t +M.T. m t +Mtor m t o r +MTOSI m t o s i +M.T.O.W. m t o w +MTPA m t p a +MTPC m t p c +MTP m t p +MTQ m t q +MTRCB m t r c b +MTRC m t r c +MTRJ m t r j +MTR m t r +M. T. S. m t s +MTS m t s +MTSU m t s u +MTTM m t t m +MTT m t t +MTU m t u +MTV m t v +MTVR m t v r +MTV's m t v's +mtvU m t v u +MTVu m t v u +MTVU m t v u +MTY m t y +MTZ m t z +MUKW m u k w +M. U. m u +MUSL m u s l +M.U.s m u's +M. U.S. m u s +MVA m v a +M. V. C. m v c +M.V.C. m v c +MVC m v c +M. V. D. m v d +MVD m v d +MVH m v h +MVK m v k +M.V.M. m v m +MVM m v m +MVMs m v m's +M. V. m v +M.V. m v +M.V.O. m v o +MVO m v o +MVP m v p +MVPs m v p's +MVPS m v p s +M.V. P.T. m v p t +MVs m v's +MVS m v s +MVSN m v s n +MVSR m v s r +MVT m v t +MVV m v v +M.W.A.M. m w a m +MWA m w a +MWC m w c +MWe m w e +M. W. E. m w e +MWF m w f +MWHL m w h l +MWIFF m w i f f +MWI m w i +MWJCHL m w j c h l +MWJHL m w j h l +M. W. J. m w j +M. W. m w +M.W. m w +M. X. m x +myb m y b +myc m y c +Myc m y c +MYC m y c +MYCN m y c n +MyDD m y d d +M. Y. m y +M.Y. m y +MySQL m y s q l +MYSQL m y s q l +MZC m z c +MZH m z h +MZM m z m +M. Z. m z +MZP m z p +MZT m z t +NAACCR n a a c c r +NAACL n a a c l +NAAC n a a c +N.A.A.C.P. n a a c p +NAACP n a a c p +NAACP's n a a c p's +N.A.C.L. n a c l +NAFBL n a f b l +NAFC n a f c +NAFC's n a f c's +NAFH n a f h +NAFI n a f i +N. A. n a +N.A. n a +NBADL n b a d l +N.B.A. n b a +NBA n b a +N.B.A.'s n b a's +NBA's n b a's +NBAs n b a's +NBCC n b c c +NBC n b c +NBC's n b c's +NBCSN n b c s n +NBDL n b d l +NBD n b d +NBFA's n b f a's +NBF n b f +NBG n b g +NBI n b i +NBK n b k +NBL n b l +N. B. n b +N.B. n b +N.B.N. n b n +NBN n b n +NBP n b p +NBQ n b q +NBR n b r +NBR's n b r's +NBSK n b s k +nbs n b s +NBS n b s +NBSP n b s p +NBTE n b t e +NBT n b t +NBTwo n b t w o +NBTY n b t y +NBW n b w +NCAAs n c a a's +NCAM n c a m +NCA n c a +NCAP n c a p +NCBC n c b c +NCBI n c b i +NCCC n c c c +NCCF n c c f +NCCMH n c c m h +NCC n c c +NCCP n c c p +NCCs n c c's +NCCS n c c s +NCCU n c c u +NCDC n c d c +NCDD's n c d d's +NCD n c d +NCES n c e s +NCFA n c f a +N.C.F. n c f +NCF n c f +NCG n c g +NCGS n c g s +NCHC n c h c +NCH n c h +NCHU n c h u +NCID n c i d +NCI n c i +NCIS n c i s +NCIS's n c i s's +NCKU n c k u +NCLB n c l b +NCLC n c l c +NCL n c l +NCLR n c l r +NCLT n c l t +NCMA n c m a +NCMC n c m c +NCMM n c m m +NCM n c m +NCMP n c m p +NCNB n c n b +N. C. n c +N.C. n c +N.C.O. n c o +NCO n c o +NCO's n c o's +NCOs n c o's +NCPA n c p a +NCPC n c p c +NCP n c p +NCPO n c p o +NCRC n c r c +NCR n c r +NCSU n c s u +NCSY n c s y +NCTA n c t a +NCTC n c t c +NCTE n c t e +NCT n c t +NCUA n c u a +NCVO n c v o +NCVS n c v s +NCVT n c v t +NCWM n c w m +NCW n c w +NDA n d a +NDE n d e +NDEP n d e p +NDFB n d f b +NDFT n d f t +N.D.G. n d g +N. D. n d +N.D. n d +N.D.N.Y. n d n y +NDPH n d p h +NDPK n d p k +N.D.P. n d p +NDP n d p +NDP's n d p's +NDRC n d r c +NDRE n d r e +NDRF n d r f +N.E.A.R. n e a r +NECW n e c w +N. E. n e +N.E. n e +N.E.R.D. n e r d +NERFU n e r f u +ner n e r +N.E.R. n e r +NER n e r +N.E.W.S.T. n e w s t +NFA n f a +NFATc n f a t c +NFAT n f a t +NFB n f b +NFB's n f b's +NFC n f c +NFC's n f c's +NFD n f d +NFEA n f e a +nfed n f e d +NFFC n f f c +NFF n f f +NFH n f h +NFHS n f h s +NFIB n f i b +NFI n f i +NFISD n f i s d +N.F.L. n f l +NFL n f l +NFLPA n f l p a +NFL's n f l's +N. F. n f +N.F. n f +NFO n f o +NFPA n f p a +NFPF n f p f +NFP n f p +NFPW n f p w +NFRS n f r s +NFS n f s +NGC n g c +Nge n g e +NGF n g f +N.G.L. n g l +N. G. n g +N.G. n g +NGO n g o +NGO's n g o's +NGOs n g o's +NGRC n g r c +N.G.R. n g r +NGR n g r +NGRR n g r r +NGS n g s +NGSS n g s s +NGST n g s t +NGTC n g t c +Nha n h a +NHA n h a +NHCEs n h c e's +NHC n h c +NHCP n h c p +NHCs n h c's +NHD n h d +NHHD n h h d +NHHI n h h i +nhi n h i +NHI n h i +NHK n h k +NHK's n h k's +N.H.L. n h l +NHM&W n h m and w +N. H. n h +N.H. n h +NHP n h p +NHPs n h p's +NHPS n h p s +NHRA n h r a +NHRC n h r c +NHSCT n h s c t +NHS n h s +NHST n h s t +NHW n h w +N.I.B.B.L.E. n i b b l e +NIBC n i b c +NIBR n i b r +N. I. n i +N.I. n i +N'I n i +N.J.A.C. n j a c +N. J. A. n j a +NJA n j a +N. J. C. n j c +NJC n j c +NJIT n j i t +N. J. M. n j m +NJM n j m +N. J. n j +N.J. n j +NJN n j n +Njoo n j o o +NJPS n j p s +NJPW n j p w +N.J.S.A. n j s a +N.J.'s n j's +NKL n k l +N. K. n k +N.K. n k +N. K. N. n k n +NKP n k p +NKP's n k p's +NKR n k r +NKT n k t +NKU n k u +nkvd n k v d +NKVD n k v d +NKVM n k v m +nkv n k v +NKX n k x +NLAES n l a e s +NLAI n l a i +NLA n l a +NLB n l b +NLCF n l c f +NLC n l c +NLCS n l c s +NLDC n l d c +NLD n l d +NLD's n l d's +NLDS n l d s +N. L. n l +N.L. n l +NLP n l p +NLRB n l r b +NLRP n l r p +NLRs n l r's +NLW n l w +NLX n l x +NMAA n m a a +NMA n m a +NMBS n m b s +NMBU n m b u +NMCA n m c a +NMCB n m c b +NMC n m c +N. M. n m +N.M. n m +NMP n m p +NMR n m r +NMRW n m r w +NMT n m t +NMU n m u +NMV n m v +NMW n m w +NNCL n n c l +NNFL n n f l +NNF n n f +N. N. n n +N.N. n n +NNSA n n s a +NNSS n n s s +NNSU n n s u +NNSW n n s w +N.O.R. n o r +NPA n p a +NPBD n p b d +NPBL n p b l +NPB n p b +NPC n p c +NPCs n p c's +NPDC n p d c +NPD n p d +NPF n p f +NPF's n p f's +NPGL n p g l +NPG n p g +NPGS n p g s +NPH n p h +NPHS n p h s +NPIM n p i m +NPI n p i +NPJ n p j +NPK n p k +NPL n p l +NPMA n p m a +NPM n p m +NPMs n p m's +N. P. n p +N.P. n p +NP n p +NPO n p o +NPP n p p +NPP's n p p's +NPRL n p r l +NPRN n p r n +NPR n p r +NPRR n p r r +NPR's n p r's +NPRSO n p r s o +NPSC n p s c +NPSG n p s g +NPSL n p s l +NP's n p's +NPs n p's +NPS n p s +NPTI n p t i +NPT n p t +NPV n p v +NPWE n p w e +NPWS n p w s +NPY n p y +NQAI n q a i +NQEA n q e a +NQHS n q h s +N. Q. n q +nri n r i +Nri n r i +NRI n r i +NRIs n r i's +NRJ n r j +NRJs n r j's +NRK n r k +NRLA n r l a +NRL n r l +NRL's n r l's +NRMA n r m a +NRM n r m +NRN n r n +nr n r +N. R. n r +N.R. n r +N.R.A. n r a +NSA n s a +NSAP n s a p +NSA's n s a's +NSB n s b +NSC n s c +NSD n s d +NSE n s e +NSFE n s f e +NSF n s f +NSFW n s f w +NSG n s g +NSI n s i +NSL n s l +NSM n s m +NSN n s n +N. S. n s +N.S. n s +NSO n s o +NSOs n s o's +NSPCC n s p c c +NSPC n s p c +NSP n s p +NSRI n s r i +NSRL n s r l +NSR n s r +NSSCD n s s c d +NSSDC n s s d c +NSS n s s +NSTA n s t a +NST n s t +NSU n s u +NSV n s v +NSVT n s v t +NSWBLF n s w b l f +NSWC n s w c +NSWCV n s w c v +NSWEC n s w e c +NSWGR n s w g r +N.S.W. n s w +NSW n s w +Nta n t a +NTA n t a +NTA's n t a's +ntb n t b +NTB n t b +NTC n t c +NTD n t d +NTFL n t f l +NTF n t f +NTI n t i +NTKF n t k f +NTL n t l +N. T. n t +N.T. n t +NTR n t r +NTR's n t r's +NTSB n t s b +NTSB's n t s b's +NTSC n t s c +NTS n t s +NTTF n t t f +NTT n t t +NTTR n t t r +NTUC n t u c +NTU n t u +NTVB n t v b +NTV n t v +ntw n t w +N.U. n u +NVA n v a +NVAO n v a o +NVC n v c +NVDA n v d a +NvDA's n v d a's +NVFC n v f c +NVGOP n v g o p +NVHJ's n v h j's +NVH n v h +NVI n v i +NVL n v l +N. V. n v +N.V. n v +NWAFU n w a f u +nwa n w a +N.W.A. n w a +NWA n w a +NWA's n w a's +N.W.F.P. n w f p +NWFP n w f p +NWMP n w m p +N&W n and w +NWN n w n +N. W. n w +N.W. n w +NWPD n w p d +N.W.P. n w p +NWP n w p +NWS n w s +N.W.T. n w t +NWT n w t +nyc n y c +N.Y.C. n y c +NYC n y c +NYC's n y c's +N.Y. G.O.P. n y g o p +N.Y.L. n y l +N. Y. n y +N.Y. n y +N.Y.P.D. n y p d +NYPD n y p d +NYPD's n y p d's +NYPL n y p l +N.Y.P. n y p +NY's n y's +N.Y.S. n y s +N.Y.S.V. n y s v +N.Y.U. n y u +NYU n y u +NYU's n y u's +NYYC n y y c +NZAID n z a i d +NZAOD n z a o d +NZCA n z c a +NZDT n z d t +NZEF n z e f +NZETC n z e t c +NZFC n z f c +NZF n z f +N'Zif n z i f +NZiK n z i k +N'Zi n z i +NZIV n z i v +NZL n z l +NZLP n z l p +N.Z. n z +NZPA n z p a +NZSAS n z s a s +NZ's n z's +NZS n z s +NZTA n z t a +N.Z.W.P.W. n z w p w +O.A.C. o a c +O. A. o a +O.A. o a +O.A.R. o a r +O.A.S. o a s +O.B.E. o b e +obl o b l +Oblt o b l t +O.B. o b +O. C. o c +O.C. o c +Octl o c t l +O.C.T. o c t +O. D. o d +O.E. o e +O.F.M. o f m +O.F. o f +ofr o f r +O.F.R. o f r +OFR o f r +O.F.T.B. o f t b +OGL o g l +O&G o and g +O. G. o g +O.G. o g +O.G.S. o g s +OGS o g s +O.H.A. o h a +O. H. o h +O.H. o h +OHV o h v +O. I. o i +O.I. o i +O. J. o j +O.J. o j +OJ o j +OJSC o j s c +OKBM o k b m +OKB o k b +OKC o k c +OK'd o k d +OKd o k d +OKD o k d +oke o k e +Oke o k e +OKH o k h +O.K.I. o k i +OKK o k k +OKM o k m +O&K o and k +O. K. o k +O.K. o k +O.L.F.A.L. o l f a l +O. L. K. o l k +O. L. o l +O.L. o l +Olo o l o +Olov o l o v +OLPC o l p c +OLPH o l p h +ols o l s +OLs o l's +OLS o l s +OLSR o l s r +OLTL o l t l +OLVT o l v t +olvwm o l v w m +olwm o l w m +OMB o m b +OMC o m c +OMCS o m c s +OMD o m d +O&MFL o and m f l +OMF o m f +omg o m g +OMG o m g +OMGs o m g's +O. M. o m +O.M. o m +OmOm o m o m +Om's o m's +OMTP o m t p +OMVG o m v g +OMW o m w +OMX o m x +ONCHR o n c h r +ONC o n c +oncu o n c u +ONDH o n d h +ond o n d +onf o n f +ONF o n f +ONGC o n g c +ONIR o n i r +ONJSC o n j s c +ONM o n m +ONMR o n m r +O. N. o n +O.N. o n +O&O o and o +OOOA o o o a +O. O. o o +O.O. o o +OO o o +OOO's o o o's +Oop o o p +OOP o o p +OOPSLA o o p s l a +oor o o r +oose o o s e +Oo's o o's +OOTP o o t p +Ootw o o t w +OOUR o o u r +OPAC o p a c +OPAG o p a g +O.P.A. o p a +OPBF o p b f +OPB o p b +OPC o p c +OPCS o p c s +OPCW o p c w +OPD o p d +O.P.I. o p i +OPIRG o p i r g +OPJHL o p j h l +OPLC o p l c +OPL o p l +OPMB o p m b +OPM o p m +OPMs o p m's +OPN o p n +O. P. o p +O.P. o p +opr o p r +Opr o p r +OPR o p r +O.R.C. o r c +ORECA o r e c a +ORFs o r f's +ORMO o r m o +ORNL o r n l +OR&N o r and n +O. R. o r +osaa o s a a +OSAA o s a a +osa o s a +Osa o s a +O.S.A. o s a +OSA o s a +O.S.B. o s b +OSB o s b +OSBs o s b's +O.S.N. o s n +OSN o s n +O. S. o s +O.S. o s +OSP o s p +OSR o s r +OSTM o s t m +O.S.T. o s t +OSTP o s t p +OSTs o s t's +osv o s v +OSV o s v +OTB o t b +OTC o t c +OTJ o t j +OTL o t l +OTMH o t m h +OTO o t o +O.T. o t +OTP o t p +OTR o t r +OTs o t's +OTS o t s +Otu o t u +OTU o t u +otv o t v +O.U. o u +OU o u +OUP o u p +OUSA o u s a +OVA's o v a's +ovca o v c a +Ovca o v c a +OVC o v c +OVC's o v c's +Ovda o v d a +OVF o v f +OVM o v m +Ovo o v o +OVO o v o +ov o v +Ov o v +O. V. o v +O.V. o v +OV o v +OVP o v p +OVS o v s +OVT o v t +OVW o v w +O.W.A. o w a +OWBT o w b t +OWCs o w c's +OWGR o w g r +OWHA's o w h a's +OWIU o w i u +OWM o w m +O. W. o w +O.W. o w +P. A. C. p a c +P. A. J. p a j +P.A.J. p a j +P. A. M. p a m +P.A.M. p a m +pa p a +P. A. p a +P.A. p a +PA p a +PAQ p a q +P.A.R. p a r +P. A. S. p a s +PBAA p b a a +PBA p b a +PBA's p b a's +PBB p b b +PBCC p b c c +pBCE p b c e +pbc p b c +P.B.C. p b c +PBF p b f +PBG p b g +PBIL p b i l +PBI p b i +PBK p b k +PBL p b l +PBM p b m +PBMR p b m r +PBO p b o +pb p b +P. B. p b +P.B. p b +PB p b +PBP p b p +PBR p b r +PBS p b s +PBT p b t +PBX p b x +PBY p b y +P. C. A. p c a +PCA p c a +PCB p c b +PCB's p c b's +PCBs p c b's +PCBS p c b s +PCC p c c +PCCS p c c s +PCCW p c c w +PCCW's p c c w's +PCDDs p c d d's +PCDHB p c d h b +PCDH p c d h +PCD p c d +PCE p c e +PCeU p c e u +PCFCL p c f c l +PCFL p c f l +PCF p c f +PCFS p c f s +PCG p c g +PCGS p c g s +PCHA p c h a +PCHA's p c h a's +P.C.H. p c h +PCH p c h +PCHR p c h r +PCIe p c i e +pci p c i +PCI p c i +PCJHL p c j h l +P. C. J. p c j +PCJ p c j +PCK p c k +PCLM p c l m +pcl p c l +PCL p c l +PCMCIA p c m c i a +PCMNO p c m n o +PCM p c m +PCMS p c m s +PCNA p c n a +PCN p c n +PcoA p c o a +PCOE p c o e +PCO p c o +pc p c +P. C. p c +P.C. p c +PC p c +PCPFL p c p f l +PCP p c p +PCPV p c p v +PCRC p c r c +PCRE p c r e +PCRev p c r e v +PCRM p c r m +PCR p c r +PCSK p c s k +PCSOM p c s o m +pcs p c s +P.C.s p c's +PC's p c's +PCTL p c t l +PCT p c t +PCTV p c t v +PCU p c u +PCU's p c u's +PCW p c w +PCX p c x +pDAB p d a b +PDAB p d a b +PDA p d a +PDAs p d a's +PDBML p d b m l +PDB p d b +pdbp p d b p +PDBsum p d b s u m +PDCI p d c i +PDC p d c +PDCPD p d c p d +PDC's p d c's +PDCs p d c's +PD&D p d and d +PDD p d d +PDE p d e +PDEs p d e's +PDES p d e s +pdf p d f +PDF p d f +PDFs p d f's +PDFT p d f t +PDGF p d g f +PDG p d g +PDHJ p d h j +PDH p d h +PDI p d i +PDJ p d j +PDK p d k +PDL p d l +PDN p d n +PDO p d o +P&D p and d +P. D. p d +P.D. p d +PD p d +PDP p d p +PDP's p d p's +PDPs p d p's +P. D. Q. p d q +PDR p d r +PDSA p d s a +PDSI p d s i +PD's p d's +PDs p d's +P.D.S. p d s +PDS p d s +PDSP p d s p +PDT p d t +P.E.I. p e i +P.E.I.'s p e i's +P.E.N. p e n +P. E. p e +P.E. p e +P.F.L. p f l +PF's p f's +PFs p f's +PFS p f s +PFU p f u +PFV p f v +PFW&C p f w and c +P.G.A. p g a +PGA p g a +PGM p g m +PGMs p g m's +PG&N p g and n +P&G p and g +P. G. p g +P.G. p g +P.G.T. p g t +PHD p h d +P. H. G. p h g +PHH p h h +PHHS p h h s +PHLF p h l f +Phlo p h l o +PHL p h l +PHN p h n +P&H p and h +P. H. p h +P.H. p h +php p h p +PHP p h p +PHPs p h p's +P.I.D.E. p i d e +P&I p and i +P. I. p i +P.I. p i +P. I. W. p i w +Piz p i z +PJB p j b +PJC p j c +PJD p j d +P. J. F. p j f +P.J.K. p j k +pj p j +P. J. p j +P.J. p j +PJ p j +PJ's p j's +PJs p j's +P.J.T. p j t +P. K. p k +P.K. p k +PK p k +PLCC p l c c +plc p l c +PLC p l c +PLC's p l c's +PLCs p l c's +P.L.O. p l o +P. L. p l +P.L. p l +PLP p l p +PLX p l x +P. M. A. p m a +PMC p m c +PMG p m g +PML p m l +PMLP p m l p +PMMA p m m a +PMMoV p m m o v +PMM p m m +PMMS p m m s +PMMT p m m t +PMOI p m o i +PMO p m o +PMPC p m p c +PMPF p m p f +p.m. p m +p.m p m +P. M. p m +P.M. p m +PM p m +PMP p m p +PMP's p m p's +PMRN p m r n +PMRO p m r o +PMR p m r +PMRR p m r r +PMSE p m s e +PMSL p m s l +PMSM p m s m +PM's p m's +PMs p m's +P. M. S. p m s +PMS p m s +PMSSY p m s s y +PMT p m t +PMVY p m v y +PMWA p m w a +PNaCl p n a c l +pna p n a +PNA p n a +Pnau p n a u +pnb p n b +PNB p n b +PNC p n c +PNE p n e +PNE's p n e's +PNEs p n e's +PNETs p n e t's +pneus p n e u s +PNFA p n f a +PNF p n f +PNGIA p n g i a +png p n g +PNG p n g +PNH p n h +PNI p n i +PNL p n l +PNL's p n l's +PNMT p n m t +PNNL p n n l +PNNs p n n's +PNoy p n o y +pn p n +P. N. p n +P.N. p n +PN p n +PNP p n p +P.N.R.A. p n r a +P.O.D. p o d +P.O.D.'s p o d's +P&O p and o +P. O. p o +P.O. p o +P.O.S. p o s +P.O.V. p o v +P.O.W. p o w +P.O.W.'s p o w's +PPACA p p a c a +PPA p p a +PPBS p p b s +PPC p p c +PPD p p d +PPE p p e +PPF p p f +PPG p p g +P&PH p and p h +PPH p p h +PPi p p i +P.P.I. p p i +PPI p p i +PPJ p p j +PPKM p p k m +PPK p p k +PPL p p l +PPM p p m +PPNB p p n b +PPN p p n +PPO p p o +P. P. p p +P.P. p p +PPRP p p r p +PPR p p r +PP's p p's +PPS p p s +PPT p p t +PPTV p p t v +PPV p p v +PPVs p p v's +PPy p p y +PQDT p q d t +PQ p q +PQQ p q q +PQ's p q's +PQS p q s +PRB p r b +PRCA p r c a +PRC p r c +PRCS p r c s +PRDM p r d m +PRD p r d +PRD's p r d's +P.R.I.M.E. p r i m e +pr p r +P. R. p r +P.R. p r +PR p r +P.S.A. p s a +P.S.C. p s c +P.S.K. p s k +PSP p s p +P. s p's +P.'s p's +Ps p's +P. S. p s +P.S. p s +PSQL p s q l +PSR p s r +PSRU p s r u +PSSA p s s a +PSSA's p s s a's +PSSAs p s s a's +PSSI p s s i +PSS p s s +PSS's p s s's +PST p s t +PSU p s u +PSV p s v +PSW p s w +pTA p t a +PTA p t a +PTA's p t a's +PTAs p t a's +PTB p t b +PTBT p t b t +PTCH p t c h +PTC p t c +Pte p t e +PTEs p t e's +PTFE p t f e +ptf p t f +PTF p t f +PTH p t h +Ptie p t i e +PTI p t i +PTI's p t i's +PTK p t k +PTLLS p t l l s +PTL p t l +PTLs p t l's +PTNNT p t n n t +Ptol p t o l +PTO p t o +PTPN p t p n +PTP p t p +pt p t +P. T. p t +P.T. p t +PT p t +PTSD p t s d +PTSE p t s e +PTS p t s +PTTGC p t t g c +PTTG p t t g +PTTOW p t t o w +PTT p t t +PTTs p t t's +PTUN p t u n +PTV p t v +PTV's p t v's +P'Twa p t w a +PTY p t y +P.U.F. p u f +Puiu p u i u +PUKKE p u k k e +PUK p u k +Pul p u l +P. U. M. p u m +PVA p v a +PVAs p v a's +PVB p v b +P.V.C. p v c +PVC p v c +PvdA p v d a +PVEM p v e m +pve p v e +PVH p v h +P.V.L. p v l +PVL p v l +PVN p v n +PVO p v o +PVP p v p +P. V. p v +P.V. p v +PV p v +pvr p v r +PVR p v r +PVRS p v r s +PVs p v's +PVS p v s +PVU p v u +PWA p w a +PWB p w b +PWC p w c +PWD p w d +PWF p w f +P. W. G. p w g +P.W.G. p w g +PWG p w g +PWI p w i +pwll p w l l +PWM p w m +P.W.O. p w o +P&W p and w +P. W. p w +P.W. p w +PW p w +Pyi p y i +PYI p y i +Pyk p y k +PYK p y k +Pyl p y l +PyL p y l +PYP p y p +P. Y. p y +Pyu p y u +Pyw p y w +Pyx p y x +PZB p z b +PZL p z l +PZP p z p +P. Z. p z +Q. A. P. q a p +QAP q a p +Q. A. q a +QA q a +Q&A q and a +Q&A's q and a's +Q&As q and a's +QbA q b a +QBE q b e +QBH q b h +QBL q b l +Q.B. q b +QB q b +Q.C.B. q c b +QCD q c d +Q.C. q c +QD q d +QDS q d s +QEA q e a +QEC q e c +Q.E.D. q e d +QED q e d +Q.E.H. q e h +Q. E. q e +Q.H.C. q h c +QHP q h p +Q.H. q h +Q.I. q i +Q. J. q j +QJ q j +Q. J. R. q j r +Q. N. q n +QPF q p f +QPFS q p f s +QPM q p m +QPO q p o +QPOs q p o's +QPP q p p +QP q p +Q.P.R. q p r +QPR q p r +QQ q q +QRL q r l +QRL's q r l's +QRNA q r n a +QROPS q r o p s +QRP q r p +QR q r +QRS q r s +QRT q r t +QRV q r v +QSAR q s a r +QSES q s e s +QSI q s i +QSL q s l +QSM q s m +QSO q s o +QST q s t +QSY q s y +QVC q v c +QVD q v d +QVM q v m +Q.V. q v +R.A.B. r a b +R. A. E. r a e +R.A.F. r a f +R.A.J. r a j +R. A. r a +R.A. r a +R&A r and a +RAV r a v +R&AW r and a w +RAZR r a z r +RBAC r b a c +RbAg r b a g +RBA r b a +RBBP r b b p +RBC r b c +RBCs r b c's +RBD r b d +RBE r b e +RBGE r b g e +RBG r b g +RBI r b i +RBIs r b i's +R. B. J. r b j +R.B.J. r b j +RBK r b k +R.B.L. r b l +RBL r b l +RBMG r b m g +RBMG's r b m g's +RBM r b m +RBMY r b m y +RBO r b o +RBP r b p +R&B r and b +R. B. r b +R.B. r b +RBR r b r +RBS r b s +RBTCO's r b t c o's +RBTH r b t h +RBWH r b w h +RBW r b w +RBX r b x +RBZ r b z +RCAC r c a c +RCAF r c a f +rca r c a +R.C.A. r c a +RCA r c a +RCA's r c a's +RCBC r c b c +RCB r c b +RCCA r c c a +RCCD r c c d +RCCL r c c l +RCCM r c c m +RCC r c c +RCI r c i +R. C. K. r c k +RCL r c l +R.C.M. r c m +RCM r c m +RCN r c n +RCPO r c p o +RCP r c p +RCPT r c p t +R&C r and c +rc r c +R. C. r c +R.C. r c +RCRD r c r d +RCR r c r +RCSB r c s b +RCS r c s +RCT r c t +RCTs r c t's +RCTS r c t s +RCTV r c t v +RCV r c v +RCW r c w +RDB r d b +RDC r d c +RDD r d d +RdE r d e +RDFC r d f c +RDF r d f +R.D.G. r d g +R&D r and d +R. D. r d +R.D. r d +R.E.A. r e a +R. E. B. r e b +R.E.D. r e d +R.E.L. r e l +R.E.M. r e m +R.E.M.'s r e m's +R.E.O. r e o +REPL r e p l +REPLs r e p l's +R.E.P. r e p +R. E. r e +R.E. r e +R.E.R. r e r +R.F.A. r f a +R.F.C. r f c +R. F. r f +R.F. r f +R. F.S. r f s +R.F.W. r f w +RFW r f w +RFX r f x +RGB r g b +RGD r g d +RGF r g f +RGG r g g +RGI r g i +RGK r g k +RGMA r g m a +RGNL r g n l +RGPH r g p h +R&G r and g +R. G. r g +R.G. r g +RGR r g r +RGS r g s +RGS's r g s's +RGTP r g t p +RGU r g u +RGV r g v +RGX r g x +R. H. C. r h c +RHC r h c +RHD r h d +RHHF r h h f +RHH r h h +RHHS r h h s +RHIBs r h i b's +RHIC r h i c +RHI r h i +Rhiw r h i w +RHK r h k +rhl r h l +RHM r h m +RhoG r h o g +Rho's r h o's +RHP r h p +R. H. r h +R.H. r h +R.H.S.J. r h s j +R. I. C. r i c +R.I.C. r i c +R. I. P. r i p +R.I.P. r i p +R&I r and i +R. I. r i +R.I. r i +riu r i u +Riu r i u +rivs r i v s +Rivu r i v u +RIXS r i x s +Rixt r i x t +RJB r j b +RJD r j d +RJE r j e +rjf r j f +RJHS r j h s +R. J. J. r j j +RJJ r j j +RJL r j l +RJN r j n +R. J. r j +R.J. r j +RJ r j +RJR r j r +RJ's r j's +RJTD r j t d +R. K. B. r k b +RKD r k d +RKI r k i +RKKA r k k a +RKL r k l +R.K.O. r k o +RKO r k o +RKO's r k o's +R. K. r k +R.K. r k +RLM r l m +RLM's r l m's +R. L. r l +R.L. r l +RL r l +RLV r l v +rly r l y +RMAF r m a f +RMAG r m a g +RMA r m a +RMB r m b +RMCH r m c h +RMCL r m c l +R.M.C. r m c +RMC r m c +RMDs r m d's +R.M.E.S. r m e s +RMFL r m f l +R.M.F. r m f +RMF r m f +RMG r m g +R. M. H. r m h +RMI r m i +RMIT r m i t +RMIT's r m i t's +RMJM r m j m +R.M.K. r m k +RMK r m k +R. M. L. r m l +RML r m l +RMLs r m l's +RMM r m m +RMP r m p +rm r m +R. M. r m +R.M. r m +RM r m +RMR r m r +RMRS r m r s +RMSD r m s d +RMSDs r m s d's +RMSE r m s e +RMs r m's +RMS r m s +R. M. W. r m w +R.M.W. r m w +RMW r m w +RMX r m x +RNAO r n a o +rna r n a +RNA r n a +RNA's r n a's +RNAs r n a's +RNAS r n a s +RNC r n c +RND r n d +RNE r n e +Rnet r n e t +RNF r n f +RNG r n g +Rnic r n i c +RNJD r n j d +RNK r n k +RNLAF r n l a f +RNLI r n l i +RNLI's r n l i's +RNNs r n n's +RNOH r n o h +RNP r n p +RNPs r n p's +RNPS r n p s +R. N. r n +R.N. r n +RN r n +R.N.R. r n r +RNZ r n z +R.O.C.K. r o c k +R.O.C. r o c +ROKMC r o k m c +ROKN r o k n +rOmpB r o m p b +Rooi r o o i +ROP r o p +R. O. r o +R.O. r o +ROVs r o v's +Roxb r o x b +Roxx r o x x +RPA r p a +RPAYC r p a y c +RPB r p b +RP&C r p and c +rpc r p c +RPC r p c +Rpe r p e +RPE r p e +RPF r p f +RPGA r p g a +rpg r p g +RPG r p g +RPG's r p g's +RPGs r p g's +RPI r p i +RPI's r p i's +RPK r p k +RPL r p l +rpm r p m +RPM r p m +RPMS r p m s +RPO r p o +RPP r p p +R. P. r p +R.P. r p +RP r p +RPs r p's +R.P.S. r p s +RPS r p s +RPT r p t +RPVE r p v e +RPV r p v +R. Q. r q +R.Q. r q +RQW r q w +rra r r a +RRA r r a +RRC r r c +RRDE r r d e +rrd r r d +Rreli r r e l i +rre r r e +RRE r r e +RRG r r g +R. R. H. r r h +RRH r r h +RRKM r r k m +R.R.K. r r k +rRNA r r n a +rRNAs r r n a's +RRN r r n +RRP r r p +R&R r and r +rr r r +R. R. r r +R.R. r r +RR r r +RRR r r r +RRSR r r s r +RRS r r s +RSAF r s a f +R. S. A. r s a +RSA r s a +RSCG r s c g +RSCJ r s c j +R.S.C. r s c +RSC r s c +RSCS r s c s +RSD r s d +RSE r s e +RSF r s f +RSFSR r s f s r +RSG r s g +RSHA r s h a +rsh r s h +RSHS r s h s +RSICC r s i c c +RSID r s i d +RSI r s i +Rsis r s i's +RSI's r s i's +RSIS r s i s +RSK r s k +RSL r s l +rsly r s l y +RsmA r s m a +RSMC r s m c +RSMI r s m i +RSML r s m l +RSM r s m +RSM's r s m's +RSNO r s n o +RSN r s n +RSNZ r s n z +R. S. O. r s o +RSO r s o +RSPB r s p b +RSPCA r s p c a +RSpec r s p e c +RSPK r s p k +RSPO r s p o +R. S. P. r s p +RSP r s p +RSR r s r +rs r s +Rs r's +R. S. r s +R.S. r s +RS r s +RSSI r s s i +R. S. S. r s s +RSS r s s +RSSSF r s s s f +RSTC r s t c +RST r s t +RSu r s u +R.S.U. r s u +RSU r s u +RSU's r s u's +RSVP r s v p +RSV r s v +RSX r s x +RTAFB r t a f b +RTA r t a +RTBF r t b f +rtb r t b +RTB r t b +RTCG's r t c g's +RTC r t c +RTCs r t c's +RTD r t d +RTEC r t e c +RTEjr r t e j r +RTeOR r t e o r +RTE r t e +RTE's r t e's +RTFB r t f b +RTFM r t f m +RTF r t f +RTF's r t f's +RTFS r t f s +RTHK r t h k +RTHK's r t h k's +RTHL r t h l +RTIP r t i p +RTI r t i +RTKL r t k l +RTK r t k +RTLM r t l m +RTL r t l +RTL's r t l's +RTML r t m l +RTMP r t m p +RTM r t m +RTN r t n +RTO r t o +RTOs r t o's +RTOS r t o s +rtPA r t p a +RTP r t p +RTR r t r +rt r t +R. T. r t +R.T. r t +RT r t +rts r t s +RTS r t s +RTS's r t s's +RTTOV r t t o v +RTT r t t +RTTY r t t y +RTUK r t u k +RTU r t u +RTVC r t v c +RTVE r t v e +RTVFBiH r t v f b i h +RTV r t v +RTVV r t v v +RTW r t w +RTXC r t x c +RTX r t x +RTZ r t z +Ruao r u a o +RUC r u c +RUC's r u c's +RUFC r u f c +Rukn r u k n +Ruk r u k +RUMC r u m c +RUNX r u n x +Rupf r u p f +Rupr r u p r +rup r u p +Rup r u p +RUP r u p +R.U.R. r u r +RUR r u r +R. U. r u +R.U. r u +RU r u +RUSD r u s d +RutB r u t b +Ruu r u u +RUV r u v +Ruwa r u w a +Ruy r u y +Ruyt r u y t +RVAR r v a r +Rvat r v a t +RVC r v c +RVCT r v c t +RVD r v d +R. V. E. r v e +RVE r v e +RVGK r v g k +R.V.G. r v g +R. V. J. r v j +R.V.J. r v j +RVM r v m +RVNG r v n g +Rvo r v o +RVO r v o +RVR r v r +R. V. r v +R.V. r v +RV r v +rvs r v s +RVTD's r v t d's +RVU r v u +rwa r w a +Rwa r w a +RWA r w a +rwb r w b +RWB r w b +RWC r w c +RWD r w d +RWDSU r w d s u +RWE r w e +RWEs r w e's +RWFC r w f c +RWIS r w i s +rwjf r w j f +R.W.R.J. r w r j +R. W. r w +R.W. r w +RW r w +RWSL r w s l +R. W. S. r w s +RWS r w s +Rxa r x a +RXL r x l +RX r x +RZR r z r +rz r z +R. Z. r z +RZ r z +RZS r z s +S.A.B. s a b +S.A.D. s a d +SAIC s a i c +Sa'id s a i d +S.A.I. s a i +SAIT's s a i t's +SAKEC s a k e c +S. A. L. s a l +S.A.P.A. s a p a +S.A.P.I. s a p i +S. A. R. s a r +S.A.R. s a r +S. A. s a +S.A. s a +SA s a +S.A.S. s a s +SAS s a s +S.A.V.A. s a v a +Saxl s a x l +Sa'yo s a y o +SAZU s a z u +SBAC's s b a c's +Sbai s b a i +S.B.A. s b a +SBA s b a +SBBK s b b k +SBB s b b +SBCM s b c m +SBCMT s b c m t +sbc s b c +SBC s b c +SBCs s b c's +SBDE s b d e +SBD s b d +SBE s b e +SBF s b f +SBI s b i +SBKP s b k p +sbk s b k +SBK s b k +SBL s b l +SBML s b m l +SBMNH s b m n h +SBM s b m +SBN s b n +SBOE s b o e +SBOL s b o l +SBOP s b o p +Sborz s b o r z +SBP s b p +SBRJ s b r j +sb s b +S. B. s b +S.B. s b +SB s b +SBSE s b s e +SBS s b s +SBS's s b s's +SBTDC s b t d c +SBT s b t +SBTU s b t u +Sbu s b u +SBU s b u +SBY s b y +SBZ s b z +S.C.A. s c a +SCA s c a +SCBA s c b a +SCB s c b +SCBWI s c b w i +SCCA s c c a +SCCA's s c c a's +SCCC s c c c +SCCI s c c i +SCCP s c c p +S.C.C. s c c +SCC s c c +scr s c r +SCR s c r +SCRs s c r's +SCRS s c r s +SCRTC s c r t c +scry s c r y +sc s c +S. C. s c +S.C. s c +SC s c +S.C.S.C. s c s c +SC's s c's +SCS s c s +SCTC s c t c +SCTE s c t e +SCTO s c t o +SCTP s c t p +SCT s c t +SCTS s c t s +SCTV s c t v +SCUAA s c u a a +Scuf s c u f +SCU's s c u's +SCW s c w +Scymn s c y m n +SD&AE s d and a e +SDAP s d a p +SDAPS s d a p s +SDA s d a +SDAS s d a s +SDASS s d a s s +SDAT s d a t +SDAX s d a x +SDB s d b +SDCC s d c c +SDCG s d c g +SDC s d c +SDDI s d d i +SDDOT s d d o t +SDD s d d +SDDS s d d s +Sdei s d e i +sde s d e +Sde s d e +SDF s d f +SDG&E s d g and e +SDHA s d h a +SDHC s d h c +SDHD s d h d +SDHHD s d h h d +SDH s d h +SDI s d i +SDJ s d j +sdk s d k +SDK s d k +SDK's s d k's +SDLC s d l c +SDLP s d l p +SDLP's s d l p's +S. D. L. s d l +S.D.N.Y. s d n y +S. D. s d +S.D. s d +SDSM&T's s d s m and t's +S. D. S. s d s +S.E.C. s e c +Sejms s e j m's +sejr s e j r +Sekl s e k l +Sek s e k +SEK s e k +S. E. s e +S.E. s e +SESL s e s l +ses s e s +Ses s e's +SEs s e's +S.E.S. s e s +SES s e s +S.E.S.'s s e s's +S.E.X. s e x +Seyh s e y h +sfadb s f a d b +SFA s f a +SFA's s f a's +Sfax s f a x +S.F.B.J. s f b j +SFB s f b +SFCC s f c c +SFCH s f c h +SFC s f c +SFDR s f d r +SFD s f d +SFE s f e +SFES s f e s +SFFAS s f f a s +SFFCo s f f c o +SFFH s f f h +SFF s f f +SFG s f g +SFI s f i +S. F. L. s f l +SFL s f l +sfn s f n +SFN s f n +SFOR s f o r +SFO s f o +SFPA s f p a +SFPD s f p d +SFP s f p +SFRA s f r a +SFRJ s f r j +SFR s f r +'sf s f +sf s f +S. F. s f +S.F. s f +SF s f +SFSG s f s g +SFSR s f s r +sfs s f s +S.F.'s s f's +SFS s f s +SFSS s f s s +SFSU s f s u +SFU s f u +SFWA s f w a +SFX s f x +SFX's s f x's +SGAE s g a e +SGAP s g a p +SGA s g a +SG&A s g and a +SGA's s g a's +sgb s g b +SGB s g b +S.G.C. s g c +SGC s g c +SGD s g d +SGE s g e +S. G. F. s g f +SGF s g f +SGH s g h +Sgip s g i p +SGI s g i +SGL s g l +sgml s g m l +SGML s g m l +SGP s g p +SGRAM s g r a m +sgra s g r a +SGR s g r +S. G. s g +S.G. s g +SGSN s g s n +SGSNs s g s n's +SGS s g s +SGSY s g s y +Sgt. sergeant +sgt s g t +SGT s g t +SGU s g u +SGV s g v +SGX s g x +SHBG s h b g +SHB s h b +SHBT s h b t +SHCA s h c a +SHC s h c +SHG s h g +SHH s h h +S.H.I.E.L.D.'s s h i e l d's +S.H.I.E.L. s h i e l +SHL s h l +Shma s h m a +SHMD s h m d +SHM s h m +SHN s h n +S. H. s h +S.H. s h +S.I.D. s i d +SIF s i f +S.I.R. s i r +S. I. s i +S.I. s i +S.I.T. s i t +SJAM s j a m +SJCH s j c h +SJC s j c +SJDA s j d a +S.J.D. s j d +sjef s j e f +SJEM s j e m +S. J. J. F. s j j f +SJK s j k +S.J.L. s j l +SJL s j l +SJM s j m +Sjon s j o n +SJPCD s j p c d +SJP s j p +SJPT s j p t +SJR s j r +SJR's s j r's +S. J. s j +S.J. s j +SJ s j +SKB s k b +SKC s k c +SKD s k d +S. K. F. s k f +SKF s k f +SKG s k g +SKH s k h +SKJ s k j +SKPC s k p c +SKP s k p +SKR s k r +SKSD s k s d +S. K. s k +S.K. s k +SK s k +S. K. S. s k s +S.L.A.A.'s s l a a's +SLAF s l a f +sla s l a +SLA s l a +SLAs s l a's +SLCO s l c o +SLC s l c +SLDL s l d l +SLD s l d +S.L.E. s l e +SLF s l f +SLG s l g +SLHS s l h s +S.L.I.F.E.R. s l i f e r +SLL s l l +SLMC s l m c +S. L. M. s l m +SLM s l m +SLN s l n +SLPIM s l p i m +SLP s l p +SLRC s l r c +SLR s l r +S&L s and l +SLS&E s l s and e +SLSF s l s f +SLSK s l s k +S. L. s l +S.L. s l +SL s l +SLS s l s +SLVR s l v r +SLV s l v +S.M.A.R.T. s m a r t +S.M.A.S.H. s m a s h +SMA s m a +SMB s m b +SMe s m e +SME s m e +SME's s m e's +SMEs s m e's +SMF s m f +SMG s m g +SMHI s m h i +SMH s m h +SmI s m i +SMI s m i +SMJR s m j r +SMK s m k +SMLS s m l s +SMMT s m m t +SMN s m n +SMNS s m n s +SMP s m p +SMP's s m p's +SMPSs s m p s's +SMPTE s m p t e +SMRJ s m r j +smr s m r +SMR s m r +S&M s and m +sm s m +S. M. s m +S.M. s m +SM s m +S.M.'s s m's +SMs s m s +S.M.S. s m s +SMS s m s +SMS's s m s's +SMSU s m s u +SMTP s m t p +S.M.T. s m t +SMT s m t +SMTV s m t v +SMU s m u +SMW s m w +SMX s m x +Smyl s m y l +SMYS s m y s +SNA s n a +SNBA s n b a +SNB s n b +SNCB s n c b +SNCC s n c c +SNCF s n c f +SNC s n c +SNDC s n d c +SND s n d +SNEP s n e p +SNESjr s n e s j r +SNES s n e s +SNET s n e t +SNF s n f +SNFU s n f u +sngle s n g l e +SNG s n g +SNK s n k +SNLA s n l a +SNL s n l +SNLS s n l s +SNMCMG s n m c m g +SNMC s n m c +SNMMA s n m m a +SNMP s n m p +SNNPR s n n p r +SNNR s n n r +SNPJ s n p j +SNP s n p +SNPs s n p's +SNRI s n r i +SNR s n r +SNSD's s n s d's +SnSe s n s e +S. N. s n +S.N. s n +SN s n +SNS s n s +S.O.E. s o e +S. O. s o +S.O. s o +Sos s o's +S.O.S. s o s +SOS s o s +Sovn s o v n +Sov s o v +SOX s o x +S. O. Y. s o y +S&P 500 s and p five hundred +S.P.A.L. s p a l +SPB s p b +SPCAs s p c a's +SPCG s p c g +SPCK s p c k +S.P.C. s p c +SPC s p c +SPDI s p d i +SPD s p d +Spe s p e +SPE s p e +SPES s p e s +SPFH s p f h +SPFL s p f l +SPF s p f +SPG s p g +SPHL s p h l +SPH s p h +SPINE's s p i n e's +SPIN's s p i n's +SPIR s p i r +SPI s p i +S.P.I.T. s p i t +SPJA s p j a +SPK s p k +SPLA s p l a +SPLC s p l c +SPL s p l +SPME s p m e +SPML s p m l +SPMRL s p m r l +SPM s p m +SPNJ s p n j +SPNM s p n m +SPNN s p n n +SPN s p n +Spoa s p o a +SPOC s p o c +SPOF s p o f +SPO s p o +SPOU s p o u +SpPIn s p p i n +SPP s p p +SPP's s p p's +SPQA s p q a +SPRL s p r l +SPRM s p r m +SPR s p r +SPRU s p r u +S&P s and p +SPSA s p s a +SPSL s p s l +S. P. s p +S.P. s p +SP s p +SP's s p's +SPs s p's +SPS s p s +SPSS s p s s +SPTA s p t a +SPT s p t +SPUC s p u c +SPUP s p u p +SPU s p u +SPUs s p u's +SPV s p v +SQA s q a +SQBB s q b b +SQI s q i +sql s q l +SQL s q l +SQM s q m +sq s q +S. Q. s q +SQ s q +Sra s r a +SRA s r a +SRAs s r a's +Srba s r b a +SRBIJA s r b i j a +Srbi s r b i +SRBOC s r b o c +SRBP s r b p +SRB s r b +SRBs s r b's +SRBY s r b y +SRCC s r c c +SRCL s r c l +SRC s r c +SRC's s r c's +SRCS s r c s +S.R.E. s r e +SRE s r e +SRFC s r f c +S.R.F. s r f +SRF s r f +S.R.G. s r g +SRG s r g +SRGs s r g's +Srhir s r h i r +SRH s r h +S. R. J. s r j +SRK s r k +SRK's s r k's +SRLGs s r l g's +S.R.L. s r l +SRL s r l +SRMC s r m c +S.R.M. s r m +SRM s r m +SRMs s r m's +Srni s r n i +SRN s r n +srp s r p +SRPT s r p t +S. R. R. s r r +SRR s r r +sr s r +S. R. s r +S.R. s r +SR s r +SRS s r s +SRT s r t +SRT's s r t's +Srul s r u l +SRU s r u +SRU's s r u's +SRV s r v +srx s r x +SRY s r y +SSAA s s a a +SSAB s s a b +SSAC s s a c +SSAR s s a r +SSA s s a +SSAs s s a's +SSBSE s s b s e +S.S.B. s s b +SSB s s b +SSCCC s s c c c +S.S.C. s s c +SSC s s c +SSCV s s c v +SSDB s s d b +SSDL s s d l +SSDP s s d p +S.S.D. s s d +SSD s s d +SSDs s s d's +SSE s s e +SSFL s s f l +SSF s s f +SSGRC s s g r c +SSG s s g +SSHSA's s s h s a's +SSH s s h +SSI s s i +SSIS s s i s +SSJA s s j a +SSKI's s s k i's +SSK s s k +SSLAM s s l a m +ssl s s l +SSL s s l +SSLT s s l t +SSME s s m e +SSM s s m +SSN s s n +SSOD s s o d +SSOF s s o f +SSO s s o +SSoSV s s o s v +SSPA s s p a +SSPH s s p h +SSPR s s p r +SSP s s p +SSPX s s p x +SSQ s s q +SSRAA s s r a a +SSRF s s r f +SSRI s s r i +SSRIs s s r i's +SSRN s s r n +SSRP s s r p +SSR s s r +S&S s and s +SSSC s s s c +SSSI s s s i +SSSR s s s r +ss s s +S. s s's +Ss s's +S. S. s s +S.S. s s +SSS s s s +SSTH s s t h +SSTL s s t l +SSTR s s t r +SST s s t +Ssu s s u +SSU s s u +SSVC s s v c +SSV s s v +SSWAHS s s w a h s +SSWC s s w c +ssw s s w +SSW s s w +SSX s s x +Ssy s s y +ST&AJ s t and a j +S.T.A.L.K.E.R. s t a l k e r +S.T.A.T.U.S. s t a t u s +STB s t b +STCC s t c c +S.T.C. s t c +STC s t c +STCW s t c w +STDP s t d p +STDs s t d's +STD s t d +STF s t f +stfv s t f v +STGs s t g's +STG s t g +STIs s t i's +STIS s t i s +STi s t i +STI s t i +STK s t k +stl s t l +S.T.L. s t l +STL s t l +STMB s t m b +STMIK s t m i k +STM's s t m's +STM s t m +STNS s t n s +STN s t n +STPI s t p i +STPNS s t p n s +STPRI s t p r i +STP s t p +S.T.R.I.D.E. s t r i d e +STRV s t r v +Stryj s t r y j +STRZ s t r z +S&T s and t +STScI s t s c i +ST's s t's +STS s t s +S. T. s t +S.T. s t +STT s t t +STVL s t v l +STVS s t v s +STV s t v +STXBP s t x b p +STX s t x +S.U.C. s u c +SUDV s u d v +suo s u o +Suo s u o +S. U. s u +SUSV s u s v +Susz s u s z +SUTs s u t's +SUT s u t +Suu s u u +SUVF s u v f +SUVs s u v's +SUV s u v +SUWN s u w n +SUW s u w +Suy s u y +Svac s v a c +svar s v a r +SVA s v a +SVB s v b +SVC s v c +SVD s v d +sve s v e +Sve s v e +SVGA s v g a +SVG s v g +SVGT s v g t +SVIA s v i a +SVM's s v m's +SVMs s v m's +SVM s v m +svn s v n +svom s v o m +SVOPC s v o p c +SVO s v o +SVP s v p +SVPW s v p w +SVSCEP s v s c e p +SVSC s v s c +SVS s v s +sv s v +S. V. s v +S.V. s v +SV s v +SVTs s v t's +SVT s v t +S.V.U. s v u +SVU s v u +SVV s v v +SVW s v w +SVYASA s v y a s a +SVZ s v z +SWABC s w a b c +SWAC s w a c +S.W.A. s w a +SWC s w c +SWD s w d +S.W.E.A.T. s w e a t +Swe s w e +SWE s w e +SWF s w f +SWG s w g +SWHL s w h l +SWHS s w h s +SWH s w h +SWJN s w j n +S. W. K. s w k +SWMRS s w m r s +SWM s w m +SWNH s w n h +SWOC s w o c +S.W.O.R.D. s w o r d +SWPA s w p a +SWPL s w p l +SWP s w p +SwRI s w r i +SWR s w r +S&W s and w +SWS s w s +sw s w +S. W. s w +S.W. s w +SW s w +SWTPC s w t p c +SWT s w t +SWWTP s w w t p +SXCT s x c t +SXN s x n +SXSW s x s w +sx s x +S.X. s x +SX s x +SXY s x y +SXZ s x z +SysML s y s m l +SysRq s y s r q +Sys s y's +SYS s y s +SysV s y s v +S. Y. s y +S.Y. s y +SyT s y t +syv s y v +Syxx s y x x +SZA s z a +Szasz s z a s z +SZDSZ s z d s z +Szer s z e r +szkic s z k i c +Szklo s z k l o +Szlak s z l a k +SzMME s z m m e +Szpir s z p i r +SZSE s z s e +sz s z +S. Z. s z +SZ s z +Szu s z u +Szyk s z y k +SZZ s z z +taf t a f +Taf t a f +TAF t a f +T.A.P. t a p +T.A.S. t a s +T. A. t a +T.A. t a +T&A t and a +TAZ t a z +TBA t b a +TBCA t b c a +TBCs t b c's +TBCS t b c s +TBD t b d +TBE t b e +TBF t b f +TBHS t b h s +TBI t b i +TBK t b k +TBL t b l +TBMA t b m a +TBMM t b m m +TBMs t b m's +T.B.M. t b m +TBM t b m +TBNJ t b n j +TBN t b n +TBP t b p +TBSC t b s c +TBS t b s +T. B. t b +T.B. t b +TB t b +TBTG t b t g +TBT t b t +TBV t b v +TBWA t b w a +TBX t b x +TCAR t c a r +TCA t c a +TCC t c c +TCDB t c d b +TCDC t c d c +TCDD t c d d +TCDOG t c d o g +TCDSB t c d s b +TCEB t c e b +T.C.E. t c e +TCE t c e +TCFAQ t c f a q +TCF t c f +TCGA t c g a +TCG t c g +TCHC t c h c +TCHRD t c h r d +TCH t c h +TCiAP t c i a p +TCI t c i +TCKs t c k's +TCLP t c l p +TCL t c l +TCM t c m +TCO t c o +TCPL t c p l +TCPMP t c p m p +TCP's t c p's +TCP t c p +TCR t c r +T. C. S. t c s +TCS t c s +tc t c +T. C. t c +T.C. t c +TC t c +TCT t c t +TCU's t c u's +TCUs t c u's +tcu t c u +TCU t c u +TCV t c v +TCWC t c w c +TCWP t c w p +TCW t c w +TDA t d a +tdb t d b +TDCC t d c c +TDCi t d c i +TDCJ t d c j +TDC's t d c's +TDC t d c +TDD t d d +TDE t d e +TDF t d f +TDH t d h +TdIF t d i f +TDI t d i +TDK t d k +TDLAS t d l a s +TDMA t d m a +TDME t d m e +TDM t d m +TDO t d o +TDP t d p +TDRS t d r s +TDR t d r +TDSB's t d s b's +TD's t d's +TDs t d's +T. D. t d +T.D. t d +TDT t d t +Teatr t e a t r +TEBD t e b d +T. E. t e +T.E. t e +TEVT t e v t +TFAP t f a p +TFAS t f a s +TFA t f a +TFC t f c +TFCU t f c u +TFF t f f +TFG t f g +T.F.H. t f h +TFH t f h +TFIH t f i h +TFIID t f i i d +TFK t f k +TFL t f l +TFM t f m +TFN t f n +TFO t f o +TFP t f p +TFR t f r +TFSA t f s a +TFSI t f s i +TFSP t f s p +TFS t f s +T&F t and f +TFTC t f t c +tf t f +T. F. t f +T.F. t f +TF t f +TFTR t f t r +TFT t f t +TFVC t f v c +TFWA t f w a +TFW t f w +TGA t g a +T. G. C. t g c +TGC t g c +tge t g e +T.G.E. t g e +TGE t g e +tgf t g f +TGF t g f +TGG t g g +TGIF t g i f +TGL t g l +TGM's t g m's +TGM t g m +TGP t g p +TGS t g s +tg t g +T. G. t g +T.G. t g +TG t g +TGVs t g v's +tgv t g v +TGV t g v +THB t h b +THCF t h c f +THC t h c +T.H.E.M. t h e m +T.H.F. t h f +THF t h f +THG t h g +THHK t h h k +T.H.I.N.K. B.I.G. t h i n k b i g +THLL t h l l +THL t h l +THMP t h m p +THM t h m +THNN t h n n +Thok t h o k +T.H.P. t h p +THQ's t h q's +THQ t h q +THR's t h r's +THR t h r +ThSe t h s e +THSR t h s r +T. H. S. t h s +T. H. t h +T.H. t h +T.H.U.N.D.E.R. t h u n d e r +THW t h w +THX t h x +TIAA t i a a +T. I. A. t i a +TIFR t i f r +TIF t i f +T.I.'s t i's +T.I. t i +Tiu t i u +tiv t i v +Tiv t i v +TIYM t i y m +Tiy's t i y's +TJFR t j f r +T.J.F. t j f +T. J. J. t j j +TJJ t j j +TJP t j p +TJRC t j r c +T. J. S. t j s +T. J. t j +T.J. t j +TJ t j +t'ju t j u +TJX t j x +tjz t j z +TKAG t k a g +TKA's t k a's +TKA t k a +T. K. B. t k b +TKD t k d +TKE t k e +TKI t k i +tko t k o +Tko t k o +TKO t k o +TKP t k p +TK's t k's +TKS t k s +tk t k +T. K. t k +T.K. t k +TK t k +TKT t k t +Tlas t l a's +TLA t l a +TLC's t l c's +TLC t l c +TlCu t l c u +TLDs t l d's +TLDS t l d s +TLD t l d +TLE t l e +TLF t l f +TLG t l g +TLH t l h +tli t l i +TLI t l i +TLM t l m +TLN t l n +tlp t l p +TLP t l p +TLRs t l r's +TLR t l r +TLs t l's +TLS t l s +TLTB t l t b +tl t l +T. L. t l +T.L. t l +TL t l +Tluk t l u k +TLU's t l u's +Tluszcz t l u s z c z +TLV t l v +TMA t m a +TMBG t m b g +TMB t m b +tmc t m c +TMC t m c +TMDL t m d l +TMD t m d +TMEM t m e m +TMF t m f +TMGs t m g's +TMG t m g +TMI t m i +TMJ t m j +T. M. M. t m m +TMMTX t m m t x +TMNT t m n t +TMOK t m o k +tmos t m o s +TMO's t m o's +TMOS t m o s +TMO t m o +TMPRSS t m p r s s +TMP t m p +tmRNA t m r n a +TMRS t m r s +TMR t m r +TMSI t m s i +TMSs t m s's +T.M.s t m's +TM's t m's +TMS t m s +tm t m +T. M. t m +T.M. t m +TM t m +T.M.T. t m t +TMT t m t +TMU t m u +TMX t m x +TMZ t m z +TNA's t n a's +TNAS t n a s +tna t n a +TNA t n a +tnbc t n b c +TNCA t n c a +TNCC t n c c +TNCDSB t n c d s b +TNC t n c +TNFA t n f a +TNFRSF t n f r s f +TNFSF t n f s f +TNF t n f +TNGHT t n g h t +TNG t n g +TNIP t n i p +TNI t n i +TNL t n l +TNMM t n m m +TNMS t n m s +TNM t n m +TNNC t n n c +TNN t n n +TNO t n o +TNPL t n p l +TNP t n p +TNQ t n q +TNRIS t n r i s +TNRP t n r p +TNR t n r +TNSDL t n s d l +TNSTC t n s t c +TNS t n s +TNTAs t n t a's +tn t n +T. N. t n +T.N. t n +TN t n +TNT's t n t's +TNT t n t +T.O.'s t o's +T. O. t o +T.O. t o +TOTP t o t p +tou t o u +Tou t o u +ToU t o u +TOU t o u +TPAC t p a c +TPAO t p a o +tPA t p a +TPA t p a +TPB t p b +TPC t p c +TPEC t p e c +TPF t p f +TPG t p g +TPH t p h +TPI t p i +TPLF t p l f +TPMS t p m s +TPM t p m +TPO t p o +TPP t p p +tpr t p r +TPR t p r +TPS t p s +tp t p +T. P. t p +T.P. t p +TP t p +TPU t p u +TPVs t p v's +TPWS t p w s +TPW t p w +TQR t q r +TQS t q s +T.Q. t q +TQ t q +tra t r a +Tra t r a +TRA t r a +TRAXX t r a x x +Trbic t r b i c +T. R. B. t r b +TRB t r b +Trcek t r c e k +TRC t r c +TRD t r d +TRFA t r f a +TRF t r f +TRG t r g +TRHS t r h s +TRH t r h +T.R.I.B.E. t r i b e +TRIZ t r i z +TrkA t r k a +TRL t r l +Trmcic t r m c i c +TRMM t r m m +T.R.M.P.A. t r m p a +T. R. M. t r m +tRNA t r n a +TRNC t r n c +TRPC t r p c +TRPM t r p m +TRPP t r p p +TRP's t r p's +TRPs t r p's +TRP t r p +TRPV t r p v +TRPY t r p y +TRSC t r s c +TRSI t r s i +trs t r s +T.R.S. t r s +TRS t r s +TRSV t r s v +tr t r +T. R. t r +T.R. t r +TR t r +TRT t r t +T.R.U. t r u +truTV t r u t v +TruTV t r u t v +Trve t r v e +TRVL t r v l +TRV t r v +TRW t r w +TRX t r x +TRZ t r z +Tsa t s a +TSA t s a +TSB t s b +TSCC t s c c +TSC t s c +TSF t s f +TSG t s g +TSH t s h +TSI t s i +tsit t s i t +Tsiv t s i v +TSJDF t s j d f +TSLA t s l a +TSLC t s l c +TSL t s l +TSN t s n +T.S.O.L. t s o l +tso t s o +Tso t s o +T.S.O. t s o +TSO t s o +TSP t s p +TSRA t s r a +TSRTC t s r t c +tsr t s r +TSR t s r +TSSA t s s a +TSS t s s +TSTA t s t a +TSTC t s t c +Ts t's +T. S. t s +T.S. t s +TS t s +TST t s t +tsus t s u s +Tsvi t s v i +TSV t s v +TSW t s w +TSX t s x +TTAB t t a b +Tta t t a +TTA t t a +TTBB t t b b +TTCP t t c p +TTC's t t c's +TTC t t c +TTDC t t d c +TTD t t d +tteok t t e o k +tte t t e +TTFF t t f f +TTHS t t h s +TTH t t h +TTI t t i +TTKG t t k g +TTKST t t k s t +TTL t t l +T.T.N. t t n +ttp t t p +TTP t t p +TTR t t r +TTS t t s +T&T t and t +T. T. t t +T.T. t t +TT t t +TTU t t u +TTV t t v +TTW t t w +TUAW t u a w +TUBB t u b b +TUDCA t u d c a +TUHUS t u h u s +TUKP's t u k p's +T'uqu t u q u +TUSCC t u s c c +TUSC t u s c +T.U. t u +TU t u +Tuzk t u z k +tvaan t v a a n +Tvam t v a m +TVARK t v a r k +tva t v a +TVA t v a +TVB's t v b's +TVB t v b +TVCM t v c m +TVC t v c +TVD t v d +TVETs t v e t's +TVE t v e +TVF t v f +TVGN t v g n +TVGOS t v g o s +TVii t v i i +TVIn t v i n +TVi t v i +TVI t v i +TVL t v l +TVMK t v m k +TVM t v m +TVN t v n +TVNZ's t v n z's +TVNZ t v n z +tvo t v o +Tvo t v o +TVO t v o +TVP t v p +TVRi t v r i +tvr t v r +TVR t v r +TVSpy t v s p y +tvs t v s +TV's t v's +TVs t v's +TVS t v s +tVTA t v t a +TVT t v t +tv t v +T. V. t v +T.V. t v +TV t v +TVXQ's t v x q's +TVXQ t v x q +TVX t v x +TWAP t w a p +TWA's t w a's +Twa t w a +TWA t w a +T. W. B. t w b +TWCS t w c s +TWC t w c +TWF t w f +Twi t w i +TWI t w i +TWK t w k +T.W.P. t w p +TWP t w p +TWR t w r +TWTs t w t's +TWT t w t +T. W. t w +T.W. t w +TW t w +T.W.U. t w u +TWU t w u +TXC t x c +TxDOT t x d o t +TXII t x i i +TXJS t x j s +TXK t x k +TXMM t x m m +TXNDC t x n d c +TXP t x p +TXTC t x t c +TXT t x t +T.X. t x +TX t x +TXU t x u +tya t y a +TYC t y c +TYIB t y i b +TYIN t y i n +Tyk t y k +Tza'ar t z a a r +Tze t z e +TZL t z l +Tzrif t z r i f +Tzuh t z u h +TZUM t z u m +Tzvi t z v i +UAAP's u a a p's +UAAP u a a p +UAB's u a b's +UAB u a b +UAC u a c +UADY u a d y +UAE's u a e's +U.A.E. u a e +UAE u a e +UAF u a f +UAHC u a h c +UAIC u a i c +UAI u a i +UALR u a l r +UAL u a l +UALVP u a l v p +UAM u a m +UANL u a n l +UAN u a n +UAOC u a o c +UAP u a p +UAR's u a r's +UARS u a r s +UAR u a r +UASLP u a s l p +UASL u a s l +Ua's u a's +UAs u a's +UAS u a s +uat u a t +U.A. u a +UA u a +UAVs u a v's +UAV u a v +U.A.W. u a w +UAW u a w +Uba u b a +UBA u b a +UBBC u b b c +UBB u b b +UbcM u b c m +UBC's u b c's +UBC u b c +UBD u b d +Ube u b e +UBE u b e +ubi u b i +Ubi u b i +U.B.I. u b i +UBI u b i +UBM's u b m's +UBM u b m +UBOS u b o s +UBO u b o +UBPR u b p r +UBP u b p +UBRE u b r e +UBRS u b r s +UB's u b's +UBS u b s +UB u b +UBU u b u +UBV u b v +UBX u b x +UCAC u c a c +Ucar u c a r +UCAR u c a r +Uca u c a +UCA u c a +UCAV u c a v +UCBH u c b h +UCBTLA u c b t l a +UCB u c b +UCCE u c c e +UCCJ u c c j +UCCS u c c s +UCC u c c +UCDA u c d a +UCDP u c d p +UCD u c d +UCF's u c f's +UCF u c f +UCG u c g +UCIEP u c i e p +UCI's u c i's +UCI u c i +UCLA's u c l a's +UCLA u c l a +UCLH u c l h +UCL u c l +Ucmak u c m a k +UCMJ u c m j +UCMMA u c m m a +UCMP u c m p +UCM u c m +Ucn u c n +UCOS u c o s +UCO u c o +UCP u c p +UCREF u c r e f +UCR u c r +UCSB u c s b +UCSC u c s c +UCSD's u c s d's +UCSD u c s d +UCSF u c s f +UCSN u c s n +uCs u c's +UCS u c s +UCTI u c t i +UCT u c t +Uc u c +U.C. u c +UC u c +Ucuncu u c u n c u +UCU u c u +UCVTS u c v t s +UCW u c w +Uczta u c z t a +UDAR u d a r +UDA's u d a's +Uda u d a +UDA u d a +UDC u d c +UDF u d f +UDG u d g +UDHR u d h r +UDICHI u d i c h i +Udit u d i t +UDK u d k +UDN u d n +Udny u d n y +UDP u d p +UDRP u d r p +UDR u d r +UDS u d s +UDTs u d t's +UDT u d t +ud u d +Ud u d +U. D. u d +UD u d +udu u d u +Udu u d u +UDYCO u d y c o +UEA u e a +UEBT u e b t +UEC u e c +Ueda's u e d a's +Uéda u e acute d a +UEFA's u e f a's +UEFA u e f a +UEFI u e f i +UEFS u e f s +UEF u e f +UEGCL u e g c l +UEI u e i +UET u e t +U. E. u e +UE u e +UFABC u f a b c +UFA's u f a's +UFCO u f c o +UFC's u f c's +UFC u f c +UFCW u f c w +UFD u f d +UFE u f e +UFF u f f +UFIA u f i a +UFIP u f i p +UFI u f i +UFJ u f j +UFL's u f l's +UFL u f l +UFM u f m +UFOFU u f o f u +UFORE u f o r e +UFO's u f o's +UFOs u f o's +Ufot u f o t +ufo u f o +Ufo u f o +UFO u f o +UFPE u f p e +UFPLS u f p l s +UFPR u f p r +UFRGS u f r g s +UFRJ u f r j +UFSB u f s b +UFSM u f s m +UF's u f's +UFS u f s +UFTP u f t p +UFT u f t +U. F. u f +UF u f +UFU u f u +UFW u f w +Uga u g a +UGA u g a +UGCCWA u g c c w a +UGC u g c +Uge u g e +UGK u g k +UGME u g m e +Ugni u g n i +UGP u g p +Ugra u g r a +Ugrszke u g r s z k e +UGS u g s +UGT u g t +Ug u g +UG u g +UHD u h d +UHF u h f +Uhha u h h a +UHMW u h m w +uhn u h n +UHOA u h o a +UHPT u h p t +UHP u h p +UHRA u h r a +UHRF u h r f +UHSAA u h s a a +UHTCs u h t c's +UHTI u h t i +Uhud u h u d +U.H. u h +uhur u h u r +UIAA u i a a +Uiam u i a m +UIA u i a +UIAW u i a w +Uibh u i b h +Uible u i b l e +UIC u i c +UID u i d +UIFL u i f l +Uig u i g +UIL u i l +UIMC u i m c +UIM u i m +UIN u i n +UIP u i p +uisae u i s a e +UISD u i s d +UiS u i s +UI's u i's +UIS u i s +UiTM u i t m +uit u i t +Uit u i t +UIT u i t +UI u i +UIV u i v +Uiy u i y +UJA u j a +UJC u j c +Ujed u j e d +UJEP u j e p +Ujsag u j s a g +UJS u j s +uj u j +U.J. u j +UJ u j +Uka u k a +UKA u k a +UKCC u k c c +UKC u k c +Ukhra u k h r a +Ukic u k i c +UKIP's u k i p's +Uki u k i +UKM u k m +UKNC u k n c +Ukoh u k o h +Uko u k o +UKPC u k p c +UKRDA's u k r d a's +UKRDA u k r d a +ukr u k r +Ukr u k r +UKR u k r +UK's u k's +UKTV's u k t v's +UKTV u k t v +Ukui u k u i +u'k u k +uk u k +Uk u k +U.K. u k +UK u k +UKUP u k u p +UKVRN u k v r n +UKW u k w +Ukwu u k w u +ULC u l c +ULDB u l d b +ULEB u l e b +ULEV u l e v +ULFA u l f a +Ulic u l i c +Ull u l l +Ulms u l m's +UL's u l's +ULS u l s +UlSU u l s u +Uluj u l u j +U.L. u l +UL u l +ulus u l u s +Ulus u l u's +ULV u l v +Ulwa u l w a +Ulwe u l w e +UMBC u m b c +Umbr u m b r +UMCC u m c c +UMC u m c +UMD u m d +umelcu u m e l c u +UMe u m e +UMF u m f +UMG u m g +UMHB u m h b +UMH u m h +UMKC's u m k c's +UMKC u m k c +UMK u m k +Umla u m l a +UML u m l +Umme u m m e +UMMSM u m m s m +UMNO's u m n o's +Umno u m n o +UMNO u m n o +UMN u m n +UMO u m o +UMPP u m p p +UMP's u m p's +Umri u m r i +UM's u m's +UMTS u m t s +UMTV u m t v +UMUC u m u c +U. M. u m +U.M. u m +UMWA's u m w a's +UMWA u m w a +UMW's u m w's +UNAB u n a b +UNAFF u n a f f +UNAH u n a h +UNAMIR u n a m i r +UNAPACK's u n a p a c k's +UNB u n b +UNCAC u n c a c +UNCAF u n c a f +UNCG's u n c g's +UNCG u n c g +UNCHE u n c h e +UNCHS u n c h s +UNCMAC u n c m a c +UNCRO u n c r o +UNC's u n c's +UNCTAD u n c t a d +UNC u n c +UNCW u n c w +UNFCCC u n f c c c +UNFICYP u n f i c y p +UNFPA u n f p a +UNFP u n f p +UNF u n f +UNFWP u n f w p +UNGC u n g c +UNHCR u n h c r +UNHRC's u n h r c's +UNH u n h +UNJLC u n j l c +Unli u n l i +UNL u n l +Unlu u n l u +UNLV u n l v +UNMCK u n m c k +UNMC u n m c +UNMEE's u n m e e's +UNMEE u n m e e +UNMIH u n m i h +UNMIK u n m i k +UNMIL u n m i l +UNMOVIC u n m o v i c +unm u n m +UNM u n m +UNNC u n n c +unnd u n n d +UNODC u n o d c +UNOMIG u n o m i g +UNPA u n p a +UNPD u n p d +UNPO u n p o +UNP u n p +UNRA u n r a +UNRSGN u n r s g n +UNRWA u n r w a +UNSCOM u n s c o m +UNSCR u n s c r +UNSC's u n s c's +UNSC u n s c +UNSh u n s h +Unst u n s t +Uns u n's +U.N.'s u n's +UNSW u n s w +UNTSO u n t s o +UNTS u n t s +Unt u n t +UNT u n t +U.N. u n +UNU u n u +UNV u n v +UNWE u n w e +UNWTO u n w t o +Unz u n z +UOCl u o c l +UODA u o d a +UofM u o f m +UOHI u o h i +Uoho u o h o +UOJCA u o j c a +UOL u o l +UOMINI u o m i n i +UoMs u o m's +UOP u o p +uORF u o r f +UOR u o r +UO's u o's +UO u o +UoW u o w +UPABA u p a b a +UPBs u p b's +UPCI u p c i +UPC u p c +UPDRS u p d r s +UPDS u p d s +Upd u p d +UPD u p d +UPEC u p e c +UPFA u p f a +UPF's u p f's +Upf u p f +UPF u p f +UPGMA u p g m a +UPHSD u p h s d +UPIN u p i n +Upir u p i r +UPI's u p i's +UPI u p i +UPLA u p l a +UPLB u p l b +UPL u p l +UPMC u p m c +UPMSS u p m s s +UPND u p n d +UPNE u p n e +UPnP u p n p +UPN u p n +UPOV u p o v +UPPP u p p p +Uppu u p p u +UPRR u p r r +UPR u p r +UPSC u p s c +U.P.S.E.B. u p s e b +UPSIDC u p s i d c +UPSID u p s i d +UPSI u p s i +UPSMF u p s m f +UPTI u p t i +UPTU u p t u +U.P. u p +UPW u p w +UPX u p x +UPyD u p y d +UQAIB u q a i b +UQAM's u q a m's +UQC u q c +U.Q.P. u q p +UQP u q p +uq u q +UQ u q +URAA u r a a +URAC u r a c +URAP u r a p +URBACT u r b a c t +URBED u r b e d +Urbz u r b z +URCL u r c l +URCSA u r c s a +URCs u r c's +URC u r c +Urdd u r d d +Urei u r e i +urf u r f +Urla u r l a +URLhttp u r l h t t p +URLs u r l's +URL u r l +URNU u r n u +UROC u r o c +URRACA u r r a c a +URSB u r s b +URSEC u r s e c +URSS u r s s +URW's u r w's +URW u r w +URZ u r z +USAACE u s a a c e +USAAC u s a a c +U.S.A.A.F. u s a a f +USAAF u s a a f +USAB u s a b +USACE u s a c e +USAC u s a c +USAFE u s a f e +USAF u s a f +USAHA u s a h a +USAID's u s a i d's +USAID u s a i d +Usal u s a l +usan u s a n +Usan u s a n +USAN u s a n +usao u s a o +USAO u s a o +USAPA u s a p a +USARPS u s a r p s +USASA u s a s a +USA's u s a's +usata u s a t a +USATF u s a t f +USAT u s a t +usa u s a +.USA u s a +U.S.A. u s a +U.S.A u s a +USA u s a +USAya u s a y a +USBA u s b a +USBC's u s b c's +USBC u s b c +USBs u s b's +USB u s b +USBWA u s b w a +USCCB u s c c b +USCF u s c f +USCGAUX u s c g a u x +USCGC u s c g c +USCG u s c g +USCHS u s c h s +Uscie u s c i e +USCIRF u s c i r f +USCIS u s c i s +USCITA u s c i t a +USCOB u s c o b +USCRP u s c r p +U.S.C.'s u s c's +USC's u s c's +USCT u s c t +U.S.C. u s c +USC u s c +USDA's u s d a's +U.S.D.A. u s d a +USDA u s d a +USDoE u s d o e +USDOJ u s d o j +USDP u s d p +USDTV u s d t v +usd u s d +USD u s d +USEER u s e e r +USENIX u s e n i x +USFbA u s f b a +USFE u s f e +USFF u s f f +USFLPA u s f l p a +USFL u s f l +USFS u s f s +USF u s f +USF&WS u s f and w s +USFWS u s f w s +USFW u s f w +USGA u s g a +U.S.G.S. u s g s +USGS u s g s +U.S. H.R. u s h r +UShs u s h's +Usia u s i a +USIA u s i a +usih u s i h +USIP's u s i p's +USIP u s i p +USISL u s i s l +USIS u s i s +USIU u s i u +USJA u s j a +USJ u s j +USLES u s l e s +USLHE u s l h e +USLMRA u s l m r a +USLTA u s l t a +USL u s l +Uslu u s l u +Usmar u s m a r +USMA u s m a +USMC M u s m c +USMCR u s m c r +USMC's u s m c's +USMC u s m c +USMLE u s m l e +USML u s m l +USMMA u s m m a +USM u s m +Usna u s n a +USNA u s n a +USNG u s n g +Usnic u s n i c +USNM u s n m +USNO u s n o +USNR u s n r +USNSCS u s n s c s +USNS u s n s +USNTDP u s n t d p +U.S.N. u s n +USN u s n +USOC's u s o c's +USOC u s o c +Usoi u s o i +Usos u s o's +Uso u s o +USO u s o +USPHL u s p h l +USPO u s p o +USPPD u s p p d +U.S.P.Q. u s p q +USPSA u s p s a +USPs u s p's +USPS u s p s +USPTA u s p t a +USPTO u s p t o +U.S.P. u s p +USP u s p +USRA u s r a +USRC u s r c +USSA u s s a +USSBA u s s b a +USSB u s s b +USSD u s s d +USSF's u s s f's +USSF u s s f +USSR's u s s r's +USSR u s s r +U.S.S.R u s s r +U.S.'s u s's +US's u s's +U.S.S. u s s +USS u s s +Usti u s t i +Ustka u s t k a +USTR's u s t r's +USTR u s t r +UST's u s t's +Usui u s u i +usum u s u m +U. S. u s +U.S. u s +Usut u s u t +USVC u s v c +Usvit u s v i t +USVI u s v i +U.S.V. u s v +USWA u s w a +USWNT's u s w n t's +USWNT u s w n t +U.S.W. u s w +USW u s w +Uta's u t a's +uta u t a +Uta u t a +UTA u t a +UTB u t b +Utca u t c a +UTC u t c +Utd's u t d's +Utd u t d +utea u t e a +UTEP u t e p +Utes u t e's +UTET u t e t +UTF u t f +Uth u t h +uti u t i +Uti u t i +UTJ u t j +UTMB u t m b +UTMF u t m f +UTM u t m +Utne u t n e +Utnur u t n u r +Uto u t o +UTO u t o +UTPB u t p b +UTP u t p +utrci u t r c i +UTRGV u t r g v +UTR u t r +utsav u t s a v +Utsav u t s a v +UTSC u t s c +UTSI u t s i +UT's u t's +UTS u t s +Utsu u t s u +UTTR u t t r +U.T. u t +Utu u t u +Utva u t v a +UTVA u t v a +UTV u t v +UUA u u a +UUCP u u c p +Uuh u u h +UUKKY u u k k y +Uul u u l +Uulu u u l u +UUP u u p +Uusi u u s i +UUs u u's +Uuto u u t o +UUT u u t +Uvac u v a c +UVB u v b +UVC u v c +Uvea u v e a +UVF u v f +UVIMB u v i m b +UVI u v i +uvnitr u v n i t r +Uvo u v o +UVT u v t +UVU u v u +U.V. u v +UV u v +UWAP u w a p +Uwasa u w a s a +UWASA u w a s a +U.W.A. u w a +UWA u w a +UWB u w b +UWC u w c +UWFi u w f i +UWF u w f +UWG u w g +UWH u w h +UWI u w i +UWM's u w m's +UWP u w p +UWRF u w r f +UWSP's u w s p's +UWSU u w s u +UW's u w's +UWS u w s +UW u w +UWW u w w +VASP v a s p +VA's v a's +V. A. v a +V.A. v a +VA v a +V&A v and a +Vav v a v +VAV v a v +Vay v a y +VBA v b a +VBBS v b b s +VBE v b e +VBI v b i +VBM v b m +VBR v b r +vb v b +V. B. v b +V.B. v b +VB v b +VCA's v c a's +VCA v c a +VCCS v c c s +VCDs v c d's +VCD v c d +VCE v c e +VCH v c h +VCIT v c i t +VCI v c i +VCJ v c j +V. C. M. v c m +VCO v c o +VCP v c p +VCQ v c q +VCRs v c r's +VCR v c r +VCSELs v c s e l's +VCSEL v c s e l +VCs v c's +VCS v c s +VCTs v c t's +VCT v c t +VCU v c u +vc v c +V. C. v c +V.C. v c +VC v c +VCXIV v c x i v +VCY v c y +VDAC v d a c +vda v d a +Vda v d a +VDCs v d c's +VDC v d c +vdiq v d i q +VDJ v d j +VDL's v d l's +VDL v d l +vdm v d m +VDM v d m +VDP v d p +VDR v d r +VDSL v d s l +VDUP v d u p +VDU v d u +V. D. v d +V.D. v d +VD v d +VDV v d v +V. D. W. v d w +vez v e z +Vez v e z +VEZ v e z +V.F.A. v f a +VFA v f a +VFC v f c +VFDS v f d s +V.F.D. v f d +VFD v f d +VFIL v f i l +VFLI v f l i +VFL's v f l's +VFL v f l +VFMC v f m c +VFM v f m +vfp v f p +vfr v f r +VFR v f r +VF's v f's +VFS v f s +VFTS v f t s +VFTX v f t x +VFU v f u +V. F. v f +V.F. v f +VF v f +VFW v f w +VFX v f x +VGA v g a +V'Ger v g e r +VGIK v g i k +VGKO v g k o +VGMA v g m a +vgmdb v g m d b +VGMDB v g m d b +VGM v g m +VGo v g o +VGP v g p +VGSCs v g s c's +VGSoM v g s o m +VGTU v g t u +VGU v g u +V. G. v g +V.G. v g +VG v g +VGV v g v +VGX v g x +VHB v h b +VHC v h c +VHD v h d +VHF v h f +VHL v h l +VHP v h p +VHR v h r +VHSL v h s l +VHS v h s +V. H. v h +VH v h +V.I.C.'s v i c's +Vict v i c t +VICT v i c t +viita v i i t a +vijf v i j f +vij v i j +Vij v i j +V.I.P.s v i p's +VIPs v i p's +V.I.P. v i p +VIP v i p +V. I. v i +V.I. v i +V. J. J. v j j +VJL v j l +VJs v j's +VJTF v j t f +V. J. v j +V.J. v j +VJ v j +VKG v k g +VKIBC's v k i b c's +V. K. N. v k n +VKOS v k o s +VKS v k s +V. K. v k +V.K. v k +VK v k +Vlah v l a h +VLA v l a +VLBA v l b a +VLBI v l b i +VLB v l b +VLCCs v l c c's +VLCS v l c s +VLC v l c +VLDB v l d b +VLDL v l d l +vlei v l e i +VLE v l e +VLEX v l e x +VLF v l f +Vlijt v l i j t +VLN v l n +VLQ v l q +VLR v l r +VLSI v l s i +VLS v l s +VLT v l t +V. L. v l +V.L. v l +VL v l +VMAs v m a's +VMAT v m a t +VMA v m a +VMCAS v m c a s +VMCM v m c m +VME v m e +VMFM v m f m +VMF v m f +VMG v m g +VMI's v m i's +VMI v m i +VMK v m k +VMM v m m +VMPS v m p s +VMS v m s +VMT v m t +vm v m +V. M. v m +V.M. v m +VM v m +VNAF v n a f +VNAV v n a v +VNCH v n c h +VNC v n c +vner v n e r +V.N.E. v n e +VNI v n i +VNO v n o +VNPT v n p t +VNQDD v n q d d +VNR v n r +VNSA v n s a +VNS v n s +VNTH v n t h +VNTR v n t r +VNU v n u +V. N. v n +V.N. v n +VN v n +V.O. v o +VO v o +Vov v o v +VOX's v o x's +Voz v o z +VPAL v p a l +VPD v p d +VPH v p h +vpis v p i s +V. P. I. v p i +VPI v p i +VPLS v p l s +VPL v p l +VPM v p m +VPNs v p n's +VPN v p n +VPO's v p o's +VPP v p p +VPRO's v p r o's +VPRO v p r o +VPS v p s +vpu v p u +V. P. v p +V.P. v p +VP v p +VQT v q t +vq v q +VQ v q +vrak v r a k +VRA v r a +Vrba v r b a +Vrbuv v r b u v +VRD v r d +Vrej v r e j +vrem v r e m +VRE v r e +VRF v r f +Vrin v r i n +VRI v r i +vrj v r j +VRLA v r l a +vrn v r n +vroee v r o e e +vrou v r o u +vrouw v r o u w +Vrouw v r o u w +VRO v r o +VRR v r r +Vrsac v r s a c +VRSA v r s a +VR's v r's +VRS v r s +Vrtis v r t i's +VRT's v r t's +VRT v r t +V&R v and r +vr v r +V. R. v r +V.R. v r +VR v r +VRV v r v +VSANs v s a n's +VSAN v s a n +VSAP's v s a p's +VSATs v s a t's +VSB's v s b's +VSB v s b +VSCP v s c p +VSC v s c +VSD v s d +Vsekh v s e k h +vse v s e +VSE v s e +VSFV v s f v +VSG v s g +VSI v s i +VSM v s m +VSNL v s n l +vso v s o +VSO v s o +VSPs v s p's +VSP v s p +VSS v s s +VSTM v s t m +VST v s t +VSU v s u +vs. versus +_vs._ versus +VSV v s v +VSW v s w +VSX v s x +vsyo v s y o +VSZ v s z +VTA v t a +VTB v t b +VTCO v t c o +VTD v t d +VTEC v t e c +VTE v t eL +Vtic v t i c +VTi v t i +VTL v t l +VTM v t m +VTR v t r +VTSK v t s k +V&T's v and t's +VTS v t s +VTU v t u +V&T v and t +VTVL v t v l +V.T. v t +VT v t +VTV v t v +Vuur v u u r +VUU v u u +V.V.B. v v b +VVCS v v c s +VVD v v d +VVIPs v v i p's +VVI v v i +VVMC v v m c +VVO v v o +VVS v v s +VVT v v t +V. V. v v +V.V. v v +VV v v +VWM v w m +VWR v w r +VW's v w's +VWs v w's +VWS v w s +VW v w +VXL v x l +VXR v x r +VX v x +Vyg v y g +vyr v y r +vy v y +VY v y +vz v z +V. Z. v z +WAAAAAGH w a a a a a g h +WAAAF w a a a f +WAAC w a a c +WAAF w a a f +WAAI w a a i +WAAV w a a v +Waay w a a y +WABB w a b b +WABC w a b c +WABD w a b d +WABG's w a b g's +WABG w a b g +W. A. C. w a c +W.A.C. w a c +Wa'il w a i l +WAIO w a i o +W.A.J. w a j +Wakf w a k f +W.A.K.O. w a k o +wa'l w a l +WANFL w a n f l +WAOK w a o k +waqf w a q f +waqt w a q t +Waqt w a q t +WAQZ w a q z +WARH w a r h +W.A.R. w a r +WASC w a s c +Wasl w a s l +WASL w a s l +W. A. S. w a s +W.A.S. w a s +Watfa w a t f a +WATL w a t l +WATW w a t w +wau w a u +Wau w a u +WAU w a u +WAVF w a v f +WAVM w a v m +W. A. w a +W.A. w a +WA w a +WAWLI w a w l i +waw w a w +WAWZ w a w z +WAYH w a y h +waza w a z a +Waza w a z a +WBAL's w b a l's +WBAM w b a m +WBA w b a +WBBL w b b l +WBBM w b b m +WBBP w b b p +WBCA w b c a +WBCN w b c n +W.B.C. w b c +WBC w b c +WBCY w b c y +WBEV w b e v +WBEZ w b e z +WBFJ w b f j +WBFO w b f o +WBF w b f +WBG w b g +WBHY w b h y +WBIC w b i c +WBIL w b i l +WBJEE w b j e e +WBKO w b k o +WBK w b k +WBLQ w b l q +WBME w b m e +WBNI w b n i +WBNX w b n x +WBON w b o n +WBO w b o +WBQC w b q c +WBRC w b r c +WBRE w b r e +WBRO w b r o +WBRZ w b r z +WBs w b's +WBS w b s +WBTI w b t i +WBT's w b t's +WBT w b t +WBTW w b t w +WBTZ w b t z +WBUR w b u r +WBVE w b v e +WBV w b v +W. B. w b +W.B. w b +WB w b +WBXH w b x h +WBZB w b z b +WBZ w b z +WCAA w c a a +WCAG w c a g +WCAU w c a u +WCA w c a +WCBA w c b a +WCBN w c b n +W&CBR w and c b r +WCBR w c b r +WCBS w c b s +WCB w c b +WCCES w c c e s +WCCG w c c g +WCCO w c c o +WCC's w c c's +WCC w c c +WCDA w c d a +WCEO w c e o +WCFL w c f l +WCF w c f +WCG w c g +WCHA w c h a +WCHL's w c h l's +WCHL w c h l +WCHS w c h s +WCHV w c h v +WCIU w c i u +WCKD w c k d +WCKY w c k y +WCLN w c l n +WCLU w c l u +WCLV w c l v +WCL w c l +WCMA w c m a +WCMC w c m c +WCMFA w c m f a +WCMH w c m h +WCMR w c m r +WCM w c m +WCNB w c n b +WCOZ w c o z +WCPO w c p o +WCPW w c p w +WCRC w c r c +WCRE w c r e +WCRL w c r l +WCSA w c s a +WCSM w c s m +WCS w c s +WCTD w c t d +WCTG w c t g +W.C.T.U. w c t u +WCTU w c t u +WCTV w c t v +WCT w c t +WCTX w c t x +WCVB's w c v b's +WCVB w c v b +WCVE w c v e +WCWA w c w a +wc w c +W. C. w c +W.C. w c +WC w c +WCWC w c w c +WCW w c w +WCXR w c x r +WCYB w c y b +WCZE w c z e +WDAF w d a f +WDA w d a +WDC w d c +WDDM w d d m +WDFN w d f n +WDF w d f +WDGY w d g y +WDH w d h +WDIA w d i a +WDIF w d i f +WDIG w d i g +WDIS w d i s +WDIV w d i v +WDJT w d j t +WDLA w d l a +WDL w d l +WDM w d m +WDNN w d n n +WDNS w d n s +WDNT w d n t +WDOK w d o k +WDO w d o +WDRE w d r e +WDR w d r +WDSE w d s e +WDSN w d s n +WDSO w d s o +WDSU w d s u +WDS w d s +WDTM w d t m +WDUQ w d u q +WDVE w d v e +WDVR w d v r +W. D. w d +W.D. w d +WDWS w d w s +WDW w d w +WDXZ w d x z +WDYZ w d y z +W. E. B. w e b +W.E.B. w e b +W. E. H. w e h +W.E.H. w e h +WEVD w e v d +WEVV w e v v +W. E. w e +W.E. w e +WEZQ w e z q +WFAA w f a a +WFAL w f a l +WFAN w f a n +WFB w f b +WFCA w f c a +WFC w f c +WFDC w f d c +WFDD w f d d +WFDF w f d f +WFDU w f d u +WFD w f d +WFGF w f g f +WFHE w f h e +W.F.H. w f h +WFH w f h +WFIKKN w f i k k n +WFIL w f i l +WFIRST w f i r s t +WFL's w f l's +WFL w f l +WFLX w f l x +WFMT w f m t +WFMU w f m u +W. F. M. w f m +WFM w f m +WF&NW w f and n w +WFNZ w f n z +WFOY w f o y +WFPC w f p c +WFPF w f p f +WFPMA w f p m a +W.F.P. w f p +WFP w f p +WFSBP w f s b p +WFS w f s +WFTDA w f t d a +WFTM w f t m +WFTV w f t v +WFT w f t +WFVA w f v a +W&F w and f +WFWA w f w a +W. F. w f +W.F. w f +WF w f +WFWM w f w m +WFXR w f x r +wga w g a +WGA w g a +WGBH w g b h +WGBI w g b i +W. G. B. w g b +WGB w g b +WGCB w g c b +W.G.C. w g c +WGC w g c +WGEM w g e m +WGFT w g f t +WGGH w g g h +WGHP w g h p +WGH w g h +WGHW w g h w +WGI w g i +WGN's w g n's +WGNT w g n t +wgn w g n +WGN w g n +WGOK w g o k +WGO w g o +WGP w g p +WGPX w g p x +WGRP w g r p +WGRT w g r t +WGRV w g r v +WGSA w g s a +WGSS w g s s +WGS w g s +WGUN w g u n +wg w g +W. G. w g +W.G. w g +WG w g +WGWW w g w w +WGZR w g z r +W.H.B. w h b +WHCA w h c a +WHCB w h c b +WHCC w h c c +W. H. C. w h c +W.H.C. w h c +WHC w h c +WHDG w h d g +WHDI w h d i +W.H.D. w h d +WHEC w h e c +WHFA w h f a +WHFM w h f m +WHFS w h f s +WHHS w h h s +W.H.H. w h h +WHJA w h j a +W.H.J. w h j +WHJY w h j y +WHKT w h k t +WHL's w h l's +WHL w h l +WHNO w h n o +WHP w h p +WHRC w h r c +WHSN w h s n +WHSS w h s s +W.H.S. w h s +WHS w h s +W. H. T. w h t +WHTZ w h t z +W&H w and h +W. H. w h +W.H. w h +WH w h +Whyld w h y l d +WHYT w h y t +W.I.B.C. w i b c +Wica w i c a +wici w i c i +WIC's w i c's +WICU w i c u +Wif w i f +WIF w i f +WIJD w i j d +Wijk w i j k +WIP w i p +W.I.S.E. w i s e +W.I.T.C.H. w i t c h +Wiwa w i w a +W. I. w i +WIXY's w i x y's +WJAL w j a l +WJAR w j a r +wjaz w j a z +WJBC w j b c +WJBF w j b f +WJBK w j b k +WJBR w j b r +WJCI w j c i +WJC w j c +WJEC w j e c +WJEF w j e f +WJER w j e r +WJET w j e t +W. J. J. O. w j j o +W.J.J.O. w j j o +WJJZ w j j z +WJKA's w j k a's +WJKA w j k a +WJKS w j k s +WJLA w j l a +WJMK w j m k +WJM w j m +WJOI w j o i +WJRD w j r d +WJR w j r +WJSC w j s c +W&J's w and j's +WJTD w j t d +WJUX w j u x +W&J w and j +W. J. w j +W.J. w j +WJ w j +WJXX w j x x +WJYL w j y l +WJYM w j y m +WJZA w j z a +WJZJ w j z j +WJZ w j z +WKAG w k a g +WKAQ w k a q +WKAR w k a r +WKA w k a +WKBD w k b d +WKBF's w k b f's +WKB w k b +WKBW w k b w +W. K. C. w k c +WKDM w k d m +WKDN w k d n +WKDZ w k d z +WKFS w k f s +W.K.H. w k h +WKH w k h +WKIK w k i k +WKIP w k i p +WKKB w k k b +WKKD w k k d +WKKZ w k k z +WKLH w k l h +WKNDS w k n d s +WKN w k n +WKQI w k q i +WKRR w k r r +WKSE w k s e +WKSM w k s m +WKSU w k s u +WKTU's w k t u's +WKTU w k t u +W.K.V. w k v +W. K. w k +W.K. w k +WKXJ w k x j +WKX w k x +WKYS's w k y s's +WKYT w k y t +WKY w k y +W.L.A.K. w l a k +WLB w l b +WLCH w l c h +W&LE w and l e +Wley w l e y +WLGH w l g h +WLH w l h +WLII w l i i +WLIR w l i r +WLJY w l j y +WLKT w l k t +WLKY w l k y +WLLM w l l m +WLNP w l n p +WLNQ w l n q +WLNS w l n s +WLPA w l p a +WLPR w l p r +WLPWR w l p w r +wlrs w l r s +WLSG w l s g +WLS w l s +WLTM w l t m +WLUK's w l u k's +WLUK w l u k +WLWC w l w c +W. L. w l +W.L. w l +WL w l +WLW w l w +WLYH's w l y h's +WLYN w l y n +WLY w l y +WLZK w l z k +WMAC w m a c +WMAG w m a g +WMAP w m a p +WMAQ w m a q +WMATA's w m a t a's +WMATA w m a t a +W.M.A. w m a +WMA w m a +WMAZ w m a z +WMBG w m b g +WMCAT w m c a t +WMCA w m c a +WMDB w m d b +WMDI w m d i +WMD w m d +WMEE w m e e +WMEN w m e n +WMET w m e t +WME w m e +WMEX w m e x +WMFC w m f c +WMFM w m f m +WMFP w m f p +W.M.F. w m f +WMF w m f +WMGF w m g f +WMG w m g +WMGY w m g y +WMHG w m h g +WMHT w m h t +WMIX w m i x +WMJR w m j r +WMJZ w m j z +WMKQ w m k q +WMKW w m k w +wml w m l +WMLW w m l w +WMLY w m l y +WMMS w m m s +W. M. M. w m m +WMNF w m n f +WMN w m n +WMOB w m o b +WMO w m o +WMRA w m r a +WMRO w m r o +WMSG w m s g +W.M.S. w m s +WMS w m s +WMTS w m t s +WMT w m t +WMTW w m t w +WMUB w m u b +WMU w m u +WMVM w m v m +WMV w m v +W&M w and m +W. M. w m +W.M. w m +WM w m +WMWM w m w m +WMXB w m x b +WMX w m x +WMYE w m y e +WMYL w m y l +WNAK w n a k +W.N.B.A. w n b a +WNBA w n b a +WNBC w n b c +WNBL w n b l +WNBR w n b r +WNB w n b +WNCN w n c n +WNC w n c +WNCX w n c x +WND w n d +WNED w n e d +WNEL w n e l +WNEM w n e m +WNEP w n e p +WNET w n e t +WNEW's w n e w's +WNEW w n e w +WNFM w n f m +WNGL w n g l +WNIT w n i t +WNJN w n j n +WNJO w n j o +WNJY w n j y +WNK w n k +WNLO w n l o +WNLU w n l u +WNL w n l +WNMX w n m x +WNO w n o +WNPT w n p t +WNRN w n r n +WNR's w n r's +WNRV w n r v +WNSH w n s h +WNSR w n s r +WNTM w n t m +WNT w n t +WNUA w n u a +WNUF w n u f +WNUSP w n u s p +WNWBL w n w b l +W. N. w n +W.N. w n +WN w n +WNWO w n w o +WNWS w n w s +WNW w n w +WNYA w n y a +WNYC's w n y c's +WNYC w n y c +WNYM w n y m +WNY w n y +WNYZ w n y z +wnzaa w n z a a +WOAA w o a a +WOAK w o a k +WOA w o a +WOC w o c +W&OD w and o d +W.O.G. w o g +WOGX's w o g x's +WOJG w o j g +WOLH w o l h +WOOM w o o m +WOPC w o p c +Worh w o r h +WOSM w o s m +W. O. w o +W.O. w o +WOWOW w o w o w +WOXY w o x y +WOZN w o z n +WPAC w p a c +WPA w p a +WPBC w p b c +WPBSA w p b s a +WPBS w p b s +WPBT w p b t +W. P. C. w p c +WPC w p c +WPDH w p d h +WPD w p d +WPFF w p f f +WPFL w p f l +WPF w p f +WPGC w p g c +WPHL w p h l +WPH w p h +WPIAL w p i a l +WPI w p i +WPIX w p i x +WPLA w p l a +WPLG w p l g +WPLS w p l s +WPMW w p m w +WPO w p o +WPPT w p p t +WPP w p p +WPRI w p r i +WPRO w p r o +WPRP w p r p +WPSC w p s c +WPSL w p s l +WPS w p s +WPTB w p t b +WPTD's w p t d's +wpt w p t +WPT w p t +WPTY w p t y +W. P. U. w p u +wp w p +W. P. w p +W.P. w p +WP w p +WPXN w p x n +WPYR w p y r +WQDE w q d e +WQHT w q h t +WQIV w q i v +WQLR w q l r +WQLZ w q l z +WQMS w q m s +WQOK w q o k +WQSO w q s o +WQSX w q s x +WQ w q +WQXI w q x i +WQXR w q x r +W. R. A. S. w r a s +WRAT w r a t +WRBU w r b u +WRCB w r c b +WRCN w r c n +WRC w r c +WRDL w r d l +Wrec w r e c +WRFD w r f d +WRFK w r f k +WRFM w r f m +WRFX w r f x +WRGO w r g o +WRGP w r g p +W. R. G. w r g +WRHA w r h a +WRHC w r h c +WRHI w r h i +W.R.I. w r i +WRKA w r k a +WRKO w r k o +WRKT w r k t +W.R.K. w r k +WRKY w r k y +WRNS w r n s +WRN w r n +WRNY w r n y +WRNZ w r n z +WROJ w r o j +WROK w r o k +WROO w r o o +WRPT w r p t +WRQQ w r q q +WRTA w r t a +WRTC w r t c +WRUV w r u v +WRU w r u +WRVL w r v l +WRVS w r v s +WRWB w r w b +W. R. w r +W.R. w r +WR w r +W.R.W. w r w +WRXT w r x t +WSAR w s a r +WSAV w s a v +W.S.A. w s a +WSA w s a +WSAZ w s a z +WSBG w s b g +WSBM w s b m +WSBT w s b t +WSB w s b +WSCA's w s c a's +WSCR w s c r +W. S. C. w s c +WSC w s c +WSDD w s d d +WSDG w s d g +WSDOT w s d o t +WSDP w s d p +WSDR w s d r +WSEG w s e g +WSEK w s e k +WSE w s e +WSFA w s f a +WSFL w s f l +WSFM w s f m +WSFS w s f s +WSFX w s f x +WSGE w s g e +WSGW w s g w +WSHL w s h l +WSHU w s h u +WSH w s h +WSIPC w s i p c +WSIS w s i s +WSJM w s j m +WSJO w s j o +WSJ w s j +WSLU w s l u +WSL w s l +WSMK w s m k +W.S.M. w s m +WSM w s m +WSNJ w s n j +WSNS w s n s +WSNT w s n t +WSN w s n +WSOF w s o f +WSOPE w s o p e +WSOP's w s o p's +WSOP w s o p +WSORR w s o r r +WSO w s o +WSOY w s o y +WSPA w s p a +WSPC w s p c +WSPD w s p d +WSPN w s p n +WSPS w s p s +WSPU w s p u +WSP w s p +WSRA w s r a +WSRE's w s r e's +WSRF w s r f +WSSA w s s a +WSSCC w s s c c +WSSSA w s s s a +WSS w s s +WSTE w s t e +WSTR w s t r +WSUA w s u a +WSUD w s u d +WSUE w s u e +WSUS w s u s +WSU w s u +WSV w s v +WSVZ w s v z +WSWI w s w i +ws w s +W. S. w s +W.S. w s +WS w s +WSW w s w +WSYN w s y n +WSYX w s y x +WTAA w t a a +WTAE w t a e +WTAF w t a f +WTAG w t a g +WTAM's w t a m's +W. T. A. w t a +WTA w t a +WTAW w t a w +WTCC w t c c +W. T. C. w t c +W.T.C. w t c +WTC w t c +WTDY w t d y +WTEL w t e l +WTEV's w t e v's +WTEV w t e v +WTF w t f +WTGB w t g b +WTHR's w t h r's +WTHS w t h s +WTIU w t i u +WTI w t i +WTIX w t i x +WTKG w t k g +WTKO w t k o +WTKR w t k r +WTKS w t k s +WTLS w t l s +WTMA w t m a +WTNH w t n h +WTN w t n +WTOP w t o p +WTO's w t o's +WTOS w t o s +WTO w t o +WTPA w t p a +WTPQ w t p q +WTR w t r +WTs w t's +WTS w t s +WTTF w t t f +WTTM w t t m +WTTs w t t's +WTT w t t +WTTW w t t w +WTTZ w t t z +WTVA w t v a +WTVC w t v c +WTVD w t v d +WTVI w t v i +WTVM w t v m +WTVU w t v u +WTVW w t v w +WTVX w t v x +WTWF w t w f +W. T. w t +W.T. w t +WT w t +WTX w t x +WUAB w u a b +WUAP w u a p +WUAs w u a's +WUC w u c +Wudl w u d l +Wuhr w u h r +Wuhu w u h u +WUKB w u k b +WUPG w u p g +WUPN w u p n +WUPZ w u p z +Wu's w u's +WUSW w u s w +WUTK w u t k +WUVC w u v c +wuv w u v +W. U. w u +Wuz w u z +WVa w v a +WVBX w v b x +WVCA w v c a +WVIT w v i t +WVJC w v j c +WVJP w v j p +WVLZ w v l z +WVNI w v n i +WVNY w v n y +W. V. O. w v o +WVO w v o +WVPA w v p a +WVPO w v p o +WVSSAC w v s s a c +WVSU w v s u +WVTU's w v t u's +WVTU w v t u +WVUE w v u e +WVU w v u +WVUW w v u w +W. V. w v +W.V. w v +WV w v +WWAA w w a a +WWAMI w w a m i +WWASP w w a s p +WWA w w a +WWAX w w a x +WWBZ w w b z +WWCD w w c d +WWC w w c +WWDB's w w d b's +WWDC w w d c +WWDR w w d r +WWD w w d +WWE's w w e's +Wwe w w e +WWE w w e +WWF's w w f's +WWF w w f +WWGC w w g c +WWHT w w h t +WWHW w w h w +WWIS w w i s +WWJP w w j p +WWKB w w k b +WWKL w w k l +WWML w w m l +WWMM w w m m +WWMX w w m x +WWMY w w m y +WWN w w n +WWNW w w n w +WWNY w w n y +WWOX w w o x +WWPG w w p g +WWP w w p +WWPW w w p w +WWRFC w w r f c +WWRV w w r v +WWSS w w s s +WWS w w s +WWTI's w w t i's +WWTI w w t i +WWTN w w t n +WWU w w u +WWVA w w v a +WWVR w w v r +WWV w w v +WWWF w w w f +WWWQ's w w w q's +W. W. w w +W.W. w w +WW w w +WWW w w w +WWWW w w w w +WWYD w w y d +WXB w x b +WXCN w x c n +WXDU w x d u +WXEL w x e l +WXIL w x i l +WXIN's w x i n's +WXIN w x i n +WXJM's w x j m's +WXJ w x j +WXN w x n +WXQW w x q w +WXTG w x t g +WXW w x w +wx w x +WX w x +W.Y. w y +WY w y +wyzc w y z c +WZBC w z b c +WZFG w z f g +WZFT w z f t +WZGC w z g c +WZJQ w z j q +WZKC w z k c +WZLR w z l r +WZLX w z l x +WZRB w z r b +WZRD w z r d +WZST w z s t +W. Z. w z +WZ w z +WZX w z x +WZZM w z z m +WZZX w z z x +xbg x b g +XBMC x b m c +XBM x b m +Xbra x b r a +XBRL x b r l +XBV x b v +XCB x c b +XCE x c e +XCMS x c m s +XCOM x c o m +XCOR x c o r +xCo x c o +XCR x c r +XCT x c t +XCVI x c v i +XCV x c v +X. C. x c +XC x c +XCX x c x +xda x d a +XDDM x d d m +XDH x d h +XDK x d k +XDRs x d r's +XDR x d r +XD's x d's +XDS x d s +X. D. x d +XD x d +Xfce x f c e +XFC x f c +XFDL x f d l +XFD x f d +XFL x f l +XFM x f m +XFS x f s +XFU x f u +xf x f +X.F. x f +XF x f +XHVFC x h v f c +xh x h +X. H. x h +XH x h +XJD x j d +XJ x j +XKE x k e +XKL x k l +xk x k +XK x k +XL x l +XMB x m b +xmc x m c +xml x m l +XML x m l +XMMS x m m s +XMPP x m p p +XMP x m p +XMRV x m r v +XMSN x m s n +XM's x m's +XMU x m u +X.M. x m +XM x m +XN x n +xO x o +X.O. x o +XO x o +XPA x p a +XPD x p d +XPe x p e +XPL x p l +XPN x p n +XP's x p's +XPS x p s +XPT x p t +XPW x p w +XP x p +X.Q. x q +XQ x q +XRCC x r c c +XRCO x r c o +XRDS x r d s +XRD x r d +XRE x r e +XR x r +XSD x s d +XSi x s i +XSI x s i +XSLT x s l t +xsr x s r +XS x s +XTB x t b +XTC x t c +XTG x t g +XTwas x t w a's +xt x t +XT x t +XVAS x v a +XV x v +XWA x w a +XWB x w b +XWF x w f +XWIS x w i s +X. W. x w +XYNU x y n u +XYP x y p +xy x y +XY x y +xyz x y z +xyZ x y z +XYZ x y z +XZ x z +Y. A. y a +Y.A. y a +YBA y b a +YBL y b l +YBNL y b n l +YBN y b n +YBR y b r +YBX y b x +Y. B. y b +Y.B. y b +YB y b +YBYS y b y s +YCC y c c +YCD y c d +ycia y c i a +ycie y c i e +YCL y c l +YCO y c o +YCU y c u +YCV y c v +yc y c +Y. C. y c +Y.C. y c +YC y c +YCYW y c y w +YDA y d a +Ydby y d b y +Yde y d e +YDG y d g +YdiB y d i b +YDK y d k +YDNC y d n c +yd y d +Y. D. y d +Y.D. y d +YD y d +Y.E. y e +YFCMD y f c m d +YFCs y f c's +YFCU y f c u +YFC y f c +yfle y f l e +YF&R y f and r +YF y f +YGB y g b +YGEX y g e x +YGLA y g l a +YGL y g l +YG's y g's +yg y g +Y. G. y g +Y.G. y g +YG y g +YHA y h a +YHWH y h w h +Y. H. y h +YH y h +YIF y i f +YIG y i g +YITP y i t p +Y. I. y i +Y.I. y i +YJO y j o +yj y j +Y. J. y j +YJ y j +Ykt y k t +Y. K. y k +Y.K. y k +YK y k +YKY y k y +Yle y l e +YLE y l e +YLF y l f +yl y l +Y. L. y l +YL y l +YMCA's y m c a's +Y.M.C.A. y m c a +YMCA y m c a +YMF y m f +Ymke y m k e +Y.M.M. y m m +YMM y m m +YMO y m o +ym y m +Y. M. y m +YM y m +yndi y n d i +Yndi y n d i +yne y n e +Yngve y n g v e +Ynis y n i's +YNK y n k +YNU y n u +yn y n +Yn y n +Y. N. y n +Y.N. y n +YN y n +Y.O. y o +YO y o +ypa y p a +YPC y p c +YPF y p f +YPG y p g +YPM y p m +YPO y p o +Y.P.P. y p p +YPP y p p +Y.P.S. y p s +YPT y p t +YPU y p u +Y.P. y p +YP y p +Y. Q. y q +YQ y q +yra y r a +Yra y r a +YRC y r c +YRF y r f +YRK y r k +YRM y r m +YRT y r t +Y&R y and r +YSK'da y s k d a +YSK'ya y s k y a +YSK y s k +YSL y s l +YSRCP y s r c p +YSR y s r +YSU's y s u's +YSU y s u +Ys y's +Y.S. y s +YS y s +YSY y s y +yta y t a +YTB y t b +Ytre y t r e +YTS y t s +YTV's y t v's +YTV y t v +Y&T y and t +Y. T. y t +Y.T. y t +YT y t +Y. V. y v +YWCA y w c a +YWC y w c +Y. W. y w +Y.Y. y y +Z. A. B. z a b +Z. A. z a +Z.A. z a +ZBC z b c +ZBS z b s +ZB z b +ZCE z c e +ZCGWM z c g w m +ZCMI z c m i +ZCTA z c t a +ZCY z c y +ZC z c +Zdar z d a r +zda z d a +ZDF z d f +ZDR z d r +Z.E.R.O. z e r o +ZFC z f c +ZFK z f k +ZFM z f m +ZFP z f p +ZFS z f s +ZFX z f x +ZF z f +ZGE z g e +ZGI z g i +zg z g +Z.G. z g +ZG z g +zh z h +Z. H. z h +Z.H. z h +ZH z h +zijn z i j n +zij z i j +Zij z i j +Z. I. z i +Z. J. z j +ZKK z k k +ZKM z k m +Z.K. z k +Z.L. z l +Z.M. z m +ZNA z n a +ZNF z n f +ZNK z n k +ZnO z n o +ZNW z n w +ZPAP z p a p +Zpav z p a v +ZPAV z p a v +ZPA z p a +ZPK z p k +Z.P. z p +ZP z p +ZQ z q +ZRC z r c +ZrI z r i +ZRK z r k +ZSC z s c +ZSE z s e +ZSF z s f +ZSG's z s g's +ZSIS z s i s +ZSL z s l +ZSO z s o +ZSSS z s s s +Zsuzsa z s u z s a +ZTE z t e +ZWYCI z w y c i +Z.W. z w +Z.X.V. z x v +Z.X. z x +ZYF z y f +Z. Y. z y +Z.Y. z y +Z. Z. z z +Z.Z. z z diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr_with_pc.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr_with_pc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..44e44242fe4cf042d5683af530cc7ced6fc52c4f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/asr_with_pc.tsv @@ -0,0 +1,71 @@ +Hon. honorable +Mt. Mount +Maj. Major +Rev. Reverend +# hash +Gov. governor +vs. versus +vs versus +dept. department +vol volume +vol. volume +bldg. building +Bldg. Building +apt. apartment +Apt. Apartment +Σ sigma +η eta +κ kappa +ω omega +σ sigma +α alpha +ν nu +δ delta +ι iota +_ underscore +% percent +& ampersand +* asterisk ++ plus +/ slash += equal sign +^ circumflex +{ left brace +| vertical bar +} right brace +~ tilde +ltd limited +int'l international +$ dollar +BMW M b m w +Capt. captain +Co. company +Col. colonel +Dr. doctor +Drs. doctors +e.g. for example +e. g. for example +ES3 e s three +Esq. esquire +F.I f +FNU f n u d s a +Ft. Fort +Gen. general +i.e. that is +Jr. junior +jr. junior +Jr junior +jr junior +Ltd. limited +Lt. lieutenant +Mr. mister +Mrs. misses +Ms. miss +Sgt. sergeant +S&P 500 s and p five hundred +Uéda u e acute d a +USMC M u s m c +vs. versus +_vs._ versus +VTE v t eL +XVAS x v a diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/ipa_symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/ipa_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f5559c7116309e621659b6b4b1db91d91836abcf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/ipa_symbols.tsv @@ -0,0 +1,521 @@ +a +aoj +aəj +aː +aːʲ +aː͡j +aː͡ɨ̯ +aˤ +aˤː +a̠ +a̠ː +a̰ +a͡e +a͡i +a͡iː +a͡i̯ +a͡j +a͡o +a͡u +a͡uː +a͡u̯ +a͡w +a͡ə +a͡ɨ̯ +a͡ɪ +a͡ʊ +b +bʱ +bʲ +bː +b̥ +c +cʰ +cː +ç +d +dʲ +dː +d̥ +d̪ +d̪ʱ +d͡z +d͡zʷ +d͡zː +d͡ʑ +d͡ʒ +d͡ʒʱ +d͡ʒʲ +d͡ʒː +e +eː +eːʲ +eː͡j +ẽː +ẽ͡j̃ +e̞ +e̞ː +e̯ +e͡i +e͡iː +e͡ɨ̯ +f +fʲ +fː +h +hː +i +iəj +iəw +iʲ +iː +iːʲ +ĩː +i̥ +i̯ +i͡u +i͡ə +i͡ɛ +j +jː +j̃ +k +kʰ +kʰː +kʲ +kʲʼ +kʷ +kʷʼ +kʼ +kː +k̚ +k̚ʲ +k̟̚ +k͈ +k͡p̚ +l +lʲ +lː +l̥ +l̩ +m +mʲ +mʲː +mː +m̥ +m̩ +n +nʲ +nː +n̥ +n̩ +o +oʲ +oː +oːʲ +ò +õ͡j̃ +õ͡w̃ +o̝ +o̞ +o̞ː +o̯ +o̰ +o͡u +o͡uː +p +pʰ +pʰː +pʲ +pʷʼ +pʼ +pː +p̚ +p̚ʲ +p͈ +p͜f +p͡f +q +qʷ +qʼ +r +rʲ +rː +r̂ +r̂ː +r̥ +r̩ +s +sʰ +sʲ +sʼ +sː +s͈ +t +tʰ +tʰː +tʲ +tʷʼ +tʼ +tː +t̚ +t̪ +t̪ʰ +t͈ +t͜s +t͡s +t͡sʰ +t͡sʰː +t͡sʲ +t͡sʷ +t͡sʼ +t͡sː +t͡ɕ +t͡ɕʰ +t͡ɕ͈ +t͡ʂ +t͡ʂʼ +t͡ʃ +t͡ʃʰ +t͡ʃʰː +t͡ʃʲ +t͡ʃʷ +t͡ʃʼ +t͡ʃː +u +uəj +uʲ +uː +uːʲ +ũː +ũ͡j̃ +u̯ +u͡e +u͡i +u͡j +u͡ɔ +u͡ə +v +vʲ +vː +w +w̃ +x +xʷ +xː +y +yː +yːʲ +y̯ +z +zʲ +zː +z̥ +à +àː +á +áː +â +âː +ã +ã̠ +æ +æː +æ̀ +æ̀ː +æ̂ +æ̂ː +æ͡ɪ +æ͡ʉ +ç +è +èː +é +éː +ê +êː +ì +ìː +í +íː +î +îː +ï +ð +ò +òː +ó +óː +ô +ôː +õ +õː +õ̞ +ø +øː +øːʲ +ø̯ +ù +ùː +ú +úː +û +ûː +ā +āː +ē +ēː +ĕ +ĕ͡ə +ě +ěː +ħ +ĩ +ĩː +ī +īː +ŋ +ŋʲ +ŋ̊ +ŋ̍ +ŋ̟ +ŋ̩ +ŋ͡m +ō +ŏ +ŏ͡ə +œ +œː +œ̃ +œ͡i +œ͡iː +œ͡ʏ +ř +řː +ũ +ũː +ū +ūː +ŭ +ŭ͡ə +ǎ +ǎː +ǐ +ǐː +ǒ +ǒː +ǔ +ǔː +ǣ +ǣː +ɐ +ɐː +ɐ̃ +ɐ̃͡j̃ +ɐ̃͡w̃ +ɐ̯ +ɐ̯̯ +ɑ +ɑː +ɑ̃ +ɑ̃ː +ɒ +ɒʲ +ɒː +ɓ +ɔ +ɔː +ɔˤː +ɔ̀ +ɔ̀ː +ɔ́ +ɔ́ː +ɔ̃ +ɔ̃ː +ɔ̰ +ɔ͡i̯ +ɔ͡ə +ɔ͡ɨ̯ +ɔ͡ɪ +ɔ͡ʊ +ɕ +ɕʰ +ɕː +ɕ͈ +ɖ +ɖʱ +ɗ +ɘ +ɘː +ə +əː +əˤ +ə̀ +ə́ +ə̃ +ə̯ +ə͡u̯ +ə͡w +ə͡ɨ +ə͡ɨ̯ +ɚ +ɛ +ɛʲ +ɛː +ɛˤː +ɛ̀ +ɛ̀ː +ɛ́ +ɛ́ː +ɛ̂ +ɛ̂ː +ɛ̃ +ɛ̃ː +ɛ̄ +ɛ̄ː +ɛ̰ +ɛ͡i +ɛ͡i̯ +ɛ͡u +ɛ͡u̯ +ɛ͡ɪ +ɛ͡ʊ +ɜ +ɜː +ɝ +ɝː +ɟ +ɟː +ɟ͡ʝ +ɡ +ɡʱ +ɡʲ +ɡʷ +ɡː +ɡ̊ +ɣ +ɤ +ɥ +ɦ +ɨ +ɨəj +ɨː +ɨ̃ᵝ +ɨ̞ +ɨ̥ᵝ +ɨ̯ +ɨ͡u̯ +ɨ͡w +ɨ͡ə +ɨᵝ +ɨᵝː +ɪ +ɪː +ɪ̀ +ɪ́ +ɪ̃ +ɪ̯ +ɪ̰ +ɪ͡u̯ +ɪ͡ʊ +ɫ +ɫː +ɬ +ɬʼ +ɭ +ɮ +ɯ +ɯː +ɯ̟̃ᵝ +ɯ̟̊ᵝ +ɯ̟ᵝ +ɯ̟ᵝː +ɰ +ɰ̃ +ɰᵝ +ɱ +ɱ̩ +ɲ +ɲː +ɲ̊ +ɲ̟ +ɳ +ɴ +ɸ +ɸʷ +ɹ +ɻ +ɽ +ɽʱ +ɾ +ɾʲ +ɾː +ɾ̝̊ +ʀ +ʁ +ʁʷ +ʁː +ʂ +ʂʷ +ʃ +ʃʰ +ʃʲ +ʃʷ +ʃʷʼ +ʃʼ +ʃː +ʈ +ʈʰ +ʉ +ʉː +ʊ +ʊ̀ +ʊ́ +ʊ̃ +ʊ̯ +ʊ̯͡i +ʊ̯͡ɨ +ʊ̰ +ʋ +ʌ +ʌ̹ +ʍ +ʎ +ʏ +ʏː +ʏ̯ +ʐ +ʐʷ +ʑ +ʒ +ʒʲ +ʒʷ +ʒː +ʔ +ʔʲ +ʔʷ +ʝ +˦ˀ˥ +˦˥ +˦˧˥ +˦˩ +˧ˀ˨ +˧˦ +˧˧ +˧˨ +˧˩ +˨˩ +˨˩˦ +˨˩˨ +β +θ +χ +χʷ +χː +ḛ +ḭ +ṵ +ẽ +ẽː +ẽ̞ +‿ \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a55a04b9ab680ab621153af96c1144854a3c046c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv @@ -0,0 +1,21 @@ +Mr. mister +Mrs. misses +Dr. doctor +Drs. doctors +Co. company +Lt. lieutenant +Sgt. sergeant +St. saint +Jr. junior +Maj. major +Hon. honorable +Gov. governor +Capt. captain +Esq. esquire +Gen. general +Ltd. limited +Rev. reverend +Col. colonel +Mt. mount +Ft. fort +etc. et cetera diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bd698fad850b19e938dc65ca7c96de930bccc522 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/symbol.tsv @@ -0,0 +1,23 @@ +& and +# hash +@ at +§ section +™ trademark +® registered trademark +© copyright +_ underscore +% percent +* asterisk ++ plus +/ slash += equal sign +^ circumflex +| vertical bar +~ tilde +$ dollar +£ pound +€ euro +₩ won +¥ yen +° degree +º degree diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..758691676048f0894e2648c9732fdec92da735d9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv @@ -0,0 +1,3890 @@ +Ph.D. PHD +Hon. honorable +Mt. Mount +Maj. Major +Rev. Reverend +w/o without +a/c number account number +c/o care of +Gov. governor +vs. versus +vs versus +dept. department +prof. professor +est. estimated +vol volume +vol. volume +bldg. building +Bldg. Building +apt. apartment +Apt. Apartment +World War I World War one +World War II World War two +etc. etcetera. +SnO2 tin four oxide +dept department +HVAC H-vac +SPDR spider +ZIP zip +~ approximately +κ kappa +ω omega +α alpha +ν nu +δ delta +Δ delta +Α alpha +β beta +Β beta +χ chi +Χ chi +ε epsilon +Ε epsilon +φ phi +Φ phi +γ gamma +Γ gamma +η eta +Η eta +ι iota +Ι iota +κ kappa +Κ kappa +λ lambda +Λ lambda +μ mu +Μ mu +ν nu +Ν nu +ο omicron +Ο omicron +π pi +Π pi +θ theta +Θ theta +ρ rho +Ρ rho +σ sigma +Σ sigma +τ tau +Τ tau +υ upsilon +Υ upsilon +ω omega +Ω omega +ξ xi +Ξ xi +ψ psi +Ψ psi +ζ zeta +Ζ zeta +ltd limited +int'l international +A. D AD +A.D AD +a. d AD +a.d AD +a. d. AD +a.d. AD +B. C BC +B.C BC +b. c BC +b.c BC +A. D. AD +A.D. AD +B. C. BC +B.C. BC +b. c. BC +b.c. BC +A. A. a a +A.A. AA +A&A AANDA +A100 AONEHUNDRED +AAAs AAA's +AaB AAB +Aabo AABO +aac AAC +AACNo AACNO +AADTs AADT's +Aadu AADU +AAFld AAFLD +Aap AAP +A. B. AB +A.B. AB +Abhi ABHI +abl ABL +Abl ABL +Abp ABP +Abr ABR +abv ABV +Abz ABZ +A&C AANDC +A. C. AC +A.C. AC +A/C AC +acac ACAC +Acad ACAD +ACC&S ACC and S +Achs ACH's +Acis ACI's +acls ACLS +ACMs ACM's +AcpA ACPA +AcpB ACPB +acq ACQ +A&CR AANDCR +AcSOC ACSOC +acu ACU +AC&W AC and W +Acy ACY +A&D AANDD +Adab ADAB +A. D. AD +A.D. AD +A.D.A.M. adam +AdaSL ADASL +AdC ADC +ADCs ADC's +AD&D AD and D +AdK ADK +Adl ADL +ADLs ADL's +adm ADM +Adm ADM +admn ADMN +ADNs ADN's +AdP ADP +ADRs ADR's +ADSRs ADSR's +ADSs ADS's +Adwa ADWA +A&E AANDE +A. E. AE +A.E. AE +Aed AED +aegte AEGTE +Aeka AEKA +ael AEL +Aema AEMA +aen AEN +aere AERE +AEre AERE +AErn AERN +Aert AERT +Aes AE's +aet AET +Afa AFA +af AF +Af AF +A. F. AF +A.F. AF +Afaf AFAF +afc AFC +AFCSThe AFCSTHE +Afd AFD +AfD AFD +AfDB AFDB +Afe AFE +Aff AFF +Afi AFI +Afiq AFIQ +AFMs AFM's +AfNS AFNS +Afo AFO +Afr AFR +A. G. AG +A.G. AG +Agco AGCO +Agda AGDA +AG&G AG and G +agg AGG +Agi AGI +Agis AGI's +agli AGLI +Agn AGN +AgN AGN +AGNs AGN's +AGRs AGR's +Agsu AGSU +agt AGT +Agte AGTE +Agu AGU +A. H. AH +A.H. AH +Ahdhra AHDHRA +ahe AHE +Ahe AHE +Ahirs AHIR's +Ahk AHK +Ahsa'i AHSAI +Ahta AHTA +aht AHT +Ahu AHU +Ahva AHVA +A&I AANDI +A. I. AI +A.I. AI +Aias AIA's +aici AICI +Aicme AICME +AICs AIC's +Aiel AIEL +Aife AIFE +AiG AIG +Aigis AIGI's +Aija AIJA +Aik AIK +AIs AI's +Aist AIST +Aitu AITU +Aiud AIUD +aius AIUS +A. J. AJ +A.J. AJ +Ajba AJBA +Ajmi AJMI +Ajok AJOK +Ajsa AJSA +Aju AJU +Akaa AKAA +a.k.a. AKA +aka. AKA +aka AKA +Aka. AKA +Aka AKA +A&K AANDK +A. K. AK +A.K. AK +AKAPs AKAP's +akas AKA's +AKAs AKA's +akh AKH +Akpa AKPA +Aku AKU +A&L AANDL +A. L. AL +A.L. AL +Alh ALH +ALUs ALU's +A'ma AMA +A&M AANDM +a.m. AM +a.m AM +A. M. AM +A.M. AM +AMAs AMA's +amb AMB +Amb AMB +AmBX AMBX +AMCs AMC's +Amda AMDA +Amd AMD +AM&FM AM and FM +Amha AMHA +aml AML +Amli AMLI +Amlwch AMLWCH +Amm AMM +AM&O AM and O +Ampt AMPT +amr AMR +A&M's AANDM's +Ams' AM's +amsl AMSL +Amta AMTA +amt AMT +Amt AMT +Amts AMT's +Amu AMU +A. N. AN +A.N. AN +andFHL ANDFHL +andPPP ANDPPP +Anej ANEJ +Angti ANGTI +Angu ANGU +Anhe ANHE +Anr ANR +Anrep ANREP +A&O AANDO +A. O. AO +A.O. AO +AOCCs AOCC's +Aodh AODH +Aodla AODLA +Aogo AOGO +Aoke AOKE +Aoko AOKO +Aola AOLA +Aol AOL +Aone AONE +Aoni AONI +Aoos AOO's +aor AOR +AORs AOR's +aos AOS +AoS AOS +Aotus AOTU's +aov AOV +aovf AOVF +A&P AANDP +A. P. AP +A.P. AP +Apc APC +APCh APCH +APCs APC's +apg APG +Aph APH +Apiao APIAO +Api API +APi API +Apic APIC +APICv APICV +Apl APL +Aplu APLU +Apphttp APPHTTP +Appl APPL +appr APPR +Appts APPT's +appu APPU +Appu APPU +A&P's AANDP's +Aps AP's +APs AP's +AP&T AP and T +Aqa AQA +Aql AQL +Aqr AQR +A&R AANDR +A. R. AR +A.R. AR +ArgR ARGR +ARGs ARG's +ArH ARH +ARMv ARMV +Arnd ARND +arv ARV +Arv ARV +Arwi ARWI +As'ad ASAD +Asai ASAI +Asao ASAO +A.s A's +A. S. AS +A.S. AS +Asasp ASASP +asbl ASBL +asci ASCI +Asci ASCI +Asdis ASDI's +Asfi ASFI +Asn ASN +Aso ASO +Aspe ASPE +Asr ASR +Assn ASSN +assoc ASSOC +Assoc ASSOC +ASSPs ASSP's +Asst ASST +Aste ASTE +Astt ASTT +Aswa ASWA +A&T AANDT +atac ATAC +Atac ATAC +Atad ATAD +Ata's ATA's +Atas ATA's +A. T. AT +A.T. AT +ATBs ATB's +atc ATC +atcc ATCC +Atg ATG +ath ATH +Ath ATH +Atia's ATIA's +Ati ATI +ATi ATI +Atid ATID +Atiiq ATIIQ +Atil ATIL +Atiq ATIQ +Ativ ATIV +Atka ATKA +Atl ATL +Atli ATLI +atm ATM +Atm ATM +ATMs ATM's +Atos ATO's +ATOs ATO's +atpB ATPB +atri ATRI +Atri ATRI +A&T's AANDT's +A.T.s AT's +AT&SF AT and SF +AT&T ATANDT +attd ATTD +AT&T's ATANDTS +atv ATV +ATVs ATV's +AtxA ATXA +A. U. AU +A.U. AU +Aub AUB +Aubl AUBL +AUVs AUV's +Auw AUW +avab AVAB +A. V. AV +A.V. AV +AVCs AVC's +avg AVG +Avn AVN +Avo AVO +AvP AVP +AVRs AVR's +Avs' AV's +Avs AV's +avvo AVVO +Awa AWA +AWAs AWA's +A. W. AW +A.W. AW +Awo AWO +Awwa AWWA +A. X. AX +A. Y. AY +A.Y. AY +A. Z. AZ +A.Z. AZ +Azg AZG +Azi AZI +azm AZM +AzTV AZTV +Baad BAAD +Ba'al BAAL +Baal BAAL +Baam BAAM +Baap BAAP +ba'as BAA's +Baat BAAT +Ba'ath BAATH +B. A. BA +B.A. BA +B&A B and A +Bac BAC +bae BAE +Bae BAE +BAe BAE +Bauw BAUW +B'Av BAV +B&B B and B +bb BB +B. B. BB +B.B. BB +bbc BBC +BB&CI BB and CI +BBC&PJR BBC and PJR +BBQ'er BBQER +B&Bs B and B's +BB&S BB and S +BBSes BBSE's +BB&T BB and T +BbvCI BBVCI +bc BC +B. C. BC +B.C. BC +Bcci BCCI +B.C.'s BC's +BCs BC's +BCSic BCSIC +BdA BDA +bd BD +B. D. BD +B.D. BD +Bde BDE +bds BDS +BdU BDU +B. E. BE +B.E. BE +Bedw BEDW +bef BEF +bei BEI +Bei BEI +Beih BEIH +Beis BEI's +bej BEJ +BEMs BEM's +Ber BER +Bes BE's +bf BF +B. F. BF +B.F. BF +BFFs BFF's +bfy BFY +B. G. BG +B.G. BG +BglII BGLII +bgs BGS +Bgy BGY +Bha BHA +B&H B and H +B. H. BH +B.H. BH +BH&E BH and E +Bhe BHE +BHMs BHM's +bhttp BHTTP +B. I. BI +B.I. BI +Bie BIE +biedt BIEDT +Biem BIEM +B.I.G.'s BIG's +BiH BIH +B. J. BJ +B.J. BJ +BJPs BJP's +B.J.'s BJ's +B&K B and K +B. K. BK +B.K. BK +BKCa BKCA +B. L. BL +B.L. BL +ble BLE +bleg BLEG +blev BLEV +bli BLI +Blla BLLA +BLR&D BLR and D +BLTs BLT's +B&M B and M +B. M. BM +B.M. BM +bmd BMD +bmi BMI +BMPs BMP's +BMRs BMR's +bmt BMT +B&N B and N +bnb BNB +B. N. BN +B.N. BN +Boac BOAC +B&O B and O +B. O. BO +B.O. BO +Boc BOC +BoC BOC +BOCs BOC's +boj BOJ +Boj BOJ +B&O's B and O's +bp BP +B. P. BP +B.P. BP +bpl BPL +B&Q B and Q +B. Q. BQ +Brbic BRBIC +B. R. BR +B.R. BR +BRBs BRB's +Brne BRNE +Brno BRNO +BRTs BRT's +Brza BRZA +brzu BRZU +brzy BRZY +Brzyk BRZYK +BsaL BSAL +B. s B's +B.'s B's +Bs B's +B. S. BS +B.S. BS +bsd BSD +bt BT +B. T. BT +B.T. BT +Btry BTRY +btsan BTSAN +btus BTUS +BTUs BTU's +B.U. BU +B. V. BV +B.V. BV +Bwa BWA +B&W B and W +B. W. BW +B.W. BW +BWTs BWT's +bwwtv BWWTV +Bxa BXA +Bxe BXE +B. Y. BY +byc BYC +bygd BYGD +Byk BYK +byn BYN +Byo BYO +Byou BYOU +Byrl BYRL +bySLC BYSLC +byt BYT +B.Z. BZ +bzhed BZHED +Bzik BZIK +Bzyb BZYB +Caat CAAT +C. A. CA +C.A. CA +C&A CANDA +CA&CC CA and CC +Cadw CADW +caeca CAECA +Cae CAE +C.A.M.'s CAM's +Capt. captain +cas CAS +Cas CA's +Casc CASC +cb CB +C. B. CB +C.B. CB +cbc CBC +CB&CNS CB and CNS +CBDs CBD's +CBGBs CBGB's +CbiXS CBIXS +CB&Q CB and Q +CBs CB's +'c C +CcaA CCAA +cca CCA +Cca CCA +C&C C and C +C. C. CC +C.C. CC +ccd CCD +CCDev CCDEV +CCDs CCD's +CCEd CCED +CCi CCI +ccm CCM +CCRCs CCRC's +cct CCT +ccus CCUS +C&D C and D +cdc CDC +cd CD +C. D. CD +C.D. CD +CD&DR CD and DR +CDi CDI +cDNA CDNA +cdnas CDNAS +cDNAs CDNA's +CDOs CDO's +CDPs CDP's +CD&R CD and R +cds CDS +CDs CD's +CDSPCo CDSPCO +CdtA CDTA +CdTe CDTE +CDTi CDTI +CD&V CD and V +Cec CEC +c'e CE +ce CE +C'e CE +Ce CE +C. E. CE +C.E. CE +Ceel CEEL +Cefn CEFN +C&EI's C and EI's +C&EN C and EN +cen CEN +Cen CEN +ceo CEO +Ceol CEOL +Ceorl CEORL +Ceos CEO's +CEOs' CEO's +CEOs CEO's +cer CER +Cer CER +ces CES +Ces CE's +Cex CEX +CeX CEX +Cfa CFA +C&F C and F +cf CF +C. F. CF +C.F. CF +CFOs CFO's +CFTs CFT's +C&G C and G +C. G. CG +C.G. CG +cgl CGL +C&H C and H +C. H. CH +C.H. CH +Chhu CHHU +Chirs CHIR's +CHP'den CHPDEN +CHPs CHP's +Chy CHY +Cia CIA +ci CI +Ci CI +C. I. CI +C.I. CI +Cicic CICIC +cii CII +cim CIM +Cim CIM +Cio CIO +CIOs CIO's +Cip CIP +cis CIS +CIs CI's +cit CIT +CiTD CITD +citS CITS +CitX CITX +Ciuc CIUC +Ciu CIU +CiU CIU +Cixi CIXI +C. J. CJ +C.J. CJ +C. K. CK +C.K. CK +ckx CKX +CLBs CLB's +clc CLC +cl CL +C. L. CL +C.L. CL +Cle CLE +cllr CLLR +CL&N CL and N +Clo CLO +CL&P CL and P +CLs CL's +CLTs CLT's +Clu CLU +Clwyd CLWYD +C&MA C and MA +cm CM +C. M. CM +C.M. CM +CMCs CMC's +Cmde CMDE +cmdr CMDR +cml CML +CMMs CMM's +cmn CMN +Cmte CMTE +cn CN +C. N. CN +C.N. CN +CNMs CNM's +cnr CNR +CNs CN's +CNTs CNT's +Cnut's CNUT's +CNVs CNV's +C&O C and O +C. O. CO +C.O. CO +Co. company +Col. colonel +C&P C and P +cp CP +C. P. CP +C.P. CP +CPCs CPC's +cpo CPO +CPUs CPU's +cq CQ +CQDs CQD's +cr CR +C. R. CR +C.R. CR +Cre CRE +crk CRK +Crkva CRKVA +crkve CRKVE +CRLs CRL's +crm CRM +Crne CRNE +Crni CRNI +Crno CRNO +CRs CR's +csa CSA +Csa CSA +Csak CSAK +Csaky CSAKY +C&S C and S +cs CS +C.s C's +Cs C's +C. S. CS +C.S. CS +C&SF C and SF +csg CSG +CsI CSI +CSis CSI's +CSIs CSI's +CSOs CSO's +cspA CSPA +csp CSP +CSPs CSP's +CSSNCs CSSNC's +Csuz CSUZ +CTAs CTA's +C&T C and T +cte CTE +CTLs CTL's +ctr CTR +Ctrip CTRIP +ctv CTV +cu CU +C. U. CU +Cuyp CUYP +C&V C and V +cvcp CVCP +'cv CV +cv CV +C. V. CV +C.V. CV +CVEs CVE's +CVOs CVO's +CVs CV's +CVTs CVT's +C. W. CW +C.W. CW +Cwej CWEJ +Cwele CWELE +cwm CWM +cwrt CWRT +C.X. CX +C. Y. CY +C.Y. CY +cyl CYL +Cyn CYN +CyP CYP +Cyrl CYRL +Cys CY's +czci CZCI +cz CZ +C.Z. CZ +czy CZY +Daai DAAI +D. A. DA +D.A. DA +D&AD DANDAD +D.A.N.C.E. dance +Dav DAV +dBa DBA +db DB +D. B. DB +D.B. DB +DCCs DCC's +D&C D and C +dc DC +D. C. DC +D.C. DC +DCom DCOM +D.C.'s DC's +DCs DC's +DCUs DCU's +D&D D and D +D. D. DD +D.D. DD +DDi DDI +Ddoc DDOC +Ddu DDU +Deah DEAH +D. E. DE +D.E. DE +deg DEG +dei DEI +Dei DEI +DeI DEI +Deijm DEIJM +Deip DEIP +dez DEZ +Dez DEZ +Dfa DFA +DFCs DFC's +D. F. DF +D.F. DF +DfE DFE +DfES DFES +DFPs DFP's +DFs DF's +dfx DFX +D&G D and G +D. G. DG +D.G. DG +Dha DHA +DHBs DHB's +D&H D and H +dh DH +D. H. DH +D.H. DH +d'HEC DHEC +dhe DHE +Dheku DHEKU +Dhod DHOD +dhol DHOL +Dhol DHOL +Dhoo DHOO +Dhor DHOR +Dhou DHOU +dhr DHR +DHSs DHS's +D.I. DI +diy DIY +DiY DIY +Diyn DIYN +DIYs DIY's +Djam DJAM +dj DJ +D. J. DJ +D.J. DJ +DJ'ed DJED +Djem DJEM +Dji DJI +Djo DJO +D'Jok DJOK +djr DJR +D.J.s DJ's +DJs DJ's +DjVu DJVU +DjVus DJVU's +D. K. DK +D.K. DK +dla DLA +dl DL +D. L. DL +D.L. DL +dlia DLIA +DL&LR DL and LR +DLLs DLL's +dlo DLO +DL&W DL and W +D.M.C.'s DMC's +D&M D and M +D. M. DM +D.M. DM +DMed DMED +DM&E DM and E +DMs DM's +DMUs DMU's +Dna DNA +DnA DNA +DnaJ DNAJ +D. N. DN +D.N. DN +Dnepr DNEPR +dnes DNES +Dnes DNE's +DNFs DNF's +DNSBLs DNSBL's +dns DNS +D.O.A.'s DOA's +dod DOD +Dod DOD +DoD DOD +D. O. DO +D.O. DO +D'oh DOH +dok DOK +Dok DOK +domt DOMT +Domt DOMT +Dop DOP +dotCMS DOTCMS +DotSVN DOTSVN +Douw DOUW +dozd DOZD +d'OZ DOZ +Dozhd DOZHD +dp DP +D. P. DP +D.P. DP +dpon DPON +DProf DPROF +DPs DP's +D.Q. DQ +drc DRC +Dr. doctor +D. R. DR +D.R. DR +dri DRI +Dri DRI +DRIs DRI's +Driu DRIU +Driv DRIV +Drnis DRNI's +Dro DRO +Drs. doctors +Drska DRSKA +druj DRUJ +DsbA DSBA +DS&BB DS and BB +dsDNA DSDNA +ds DS +Ds D's +D. S. DS +D.S. DS +DSGi DSGI +dsi DSI +DSi DSI +dsl DSL +DSMs DSM's +dsn DSN +D&SNGRR D and SNGRR +dsu' DSU +DTCs DTC's +DTDs DTD's +D. T. DT +D.T. DT +DT&E DT and E +DT&I DT and I +dtl DTL +DTs DT's +dty DTY +D.U. DU +dva DVA +Dva DVA +DVCs DVC's +dvd DVD +DVDs DVD's +dv DV +D. V. DV +D.V. DV +dve DVE +dvfb DVFB +dvi DVI +dvije DVIJE +Dvin DVIN +Dvir DVIR +Dwa DWA +D&W D and W +D. W. DW +D.W. DW +Dwedw DWEDW +Dwi DWI +dwr DWR +dwur DWUR +dxa DXA +dx DX +D.X. DX +DXers DXER's +Dydd DYDD +dy DY +Dy DY +D. Y. DY +dypl DYPL +dyr DYR +dz DZ +dzis DZIS +Dzog DZOG +Dzor DZOR +Dzus DZU's +E. A. EA +E.A. EA +EAMs EAM's +eas EAS +Eas EA's +Eav EAV +E. B. EB +E.B. EB +Ebn EBN +E&BR E and BR +Ebru EBRU +EBWs EBW's +ecc ECC +Eccl ECCL +ec EC +Ec EC +E. C. EC +E.C. EC +ece ECE +Ece ECE +Ecem ECEM +ecg ECG +ecma ECMA +ECNs ECN's +ECP&DA's ECP and DA's +eCRM ECRM +ect ECT +ECVs ECV's +edb EDB +E. D. ED +E.D. ED +Edh EDH +edhe EDHE +EDMs EDM's +Edmx EDMX +edn EDN +Edn EDN +eds EDS +Eds ED's +Edw EDW +Eeb EEB +E. E. EE +E.E. EE +Eef EEF +Eega EEGA +eene EENE +eep EEP +Ees EE's +eeuw EEUW +Eeuw EEUW +EEZs EEZ's +efc EFC +ef EF +Ef EF +E. F. EF +E.F. EF +eFS EFS +Efs EF's +Efu EFU +Egba EGBA +egfl EGFL +e.g. for example +e. g. for example +EG&G EG and G +Egi EGI +E&GR E and GR +egy EGY +Egyl EGYL +EHAs EHA's +E. H. EH +E.H. EH +ehf EHF +Ehttp EHTTP +EiCs EIC's +E. I. EI +E.I. EI +eIF EIF +Eitr EITR +EITs EIT's +EJBs EJB's +ej EJ +E. J. EJ +E.J. EJ +E. K. EK +E.K. EK +eki EKI +Ekma EKMA +Ekow EKOW +eks EKS +Eks EK's +E. L. EL +E.L. EL +ELTs ELT's +ELUs ELU's +Embd EMBD +eMC EMC +EMDs EMD's +Emea EMEA +E&M E and M +E. M. EM +E.M. EM +Emge EMGE +emra EMRA +E&NA E and NA +Enas ENA's +EnBW ENBW +EncFS ENCFS +Encyc ENCYC +E. N. EN +E.N. EN +Enes ENE's +engl ENGL +Engl ENGL +Engr ENGR +Eni ENI +enn ENN +Enn ENN +ens ENS +Ens EN's +Enso ENSO +ENTJs ENTJ's +EoD EOD +E. O. EO +E.O. EO +Eois EOI's +eok EOK +Eole EOLE +Eol EOL +Eolss EOLS's +Eom EOM +eop EOP +eoptta EOPTTA +Eora EORA +Eorl EORL +Eors EOR's +E&P E and P +Epe EPE +ep EP +Ep EP +E. P. EP +E.P. EP +Eph EPH +Ephs EPH's +epi EPI +Epi EPI +ePO EPO +epos' EPO's +epos EPOS +Epos EPO's +E.P.s EP's +eq EQ +Eq EQ +E. R. ER +E.R. ER +ERJs ERJ's +Eru ERU +Eruv ERUV +Erv ERV +ES3 ES three +Esa ESA +Esam ESAM +Esa's ESA's +Esat ESAT +ESCs ESC's +Esd ESD +E. S. ES +E.S. ES +esi ESI +espn ESPN +ESPNhttp ESPNHTTP +esq ESQ +Esq ESQ +Esq. esquire +Esraa ESRAA +Esra ESRA +Esref ESREF +Esri ESRI +estd ESTD +Estd ESTD +esu ESU +Esva ESVA +Etad ETAD +ETBs ETB's +E. T. ET +E.T. ET +ETFs ETF's +Et'hem ETHEM +Eti ETI +etj ETJ +Eto ETO +ets ETS +Ets ET's +ETs ET's +ett ETT +Ett ETT +Ettre ETTRE +Etts ETT's +Etz ETZ +EUBs EUB's +E. U. EU +E.U. EU +eup EUP +Eurwg EURWG +Euse EUSE +Eusi EUSI +EvaGT EVAGT +E. V. EV +E.V. EV +evl EVL +Evna EVNA +evnt EVNT +EVs EV's +evv EVV +Ewa's EWA's +E. W. EW +E.W. EW +Ewha EWHA +EWOs EWO's +Ewu EWU +excl EXCL +exd EXD +E.X. EX +exhb EXHB +exh EXH +Exptl EXPTL +exsul EXSUL +E&Y E and Y +E. Y. EY +Eyk EYK +eyu EYU +Ezaa EZAA +ez EZ +Ez EZ +Ezh EZH +Ezu EZU +Ezy EZY +Faaa FAAA +faa FAA +Faa FAA +faama FAAMA +FabH FABH +FabR FABR +F.A.B.'s FAB's +Fadl FADL +fa FA +Fa FA +F. A. FA +F.A. FA +Faf FAF +Fafhrd FAFHRD +fai FAI +Fai FAI +FAIPs FAIP's +Faiq FAIQ +fajn FAJN +Fal FAL +farw FARW +fasc FASC +Fasc FASC +fas FAS +Fas FA's +FAs FA's +FasL FASL +F. B. FB +F.B. FB +F.B.G.s FBG's +FBOs FBO's +fcb FCB +FCBs FCB's +F&C F and C +fc FC +F. C. FC +F.C. FC +F.C.'s FC's +FD&C FD and C +FDCPAs FDCPA's +F. D. FD +F.D. FD +FDICs FDIC's +FdI FDI +F. E. FE +F.E. FE +FEGs FEG's +Fes FE's +ffc FFC +ff FF +F. F. FF +F.F. FF +ffm FFM +FFs FF's +F. G. FG +F.G. FG +fgk FGK +FGs FG's +F. H. FH +F.H. FH +fhm FHM +fiadh FIADH +fia FIA +FiBL FIBL +Fi'd FID +F.I F +F&I F and I +F. I. FI +F.I. FI +Figl FIGL +fija FIJA +Fio FIO +FiO FIO +F. J. FJ +F.J. FJ +F. K. FK +F.K. FK +Fla FLA +Fles FLE's +fl FL +F. L. FL +F.L. FL +fli FLI +flyr FLYR +fm FM +F. M. FM +F.M. FM +fMRI FMRI +FMs FM's +F. N. FN +F.N. FN +F. O. FO +FPDs FPD's +fp FP +F. P. FP +F.P. FP +FPGAs FPGA's +FPM&SA FPM and SA +fps FPS +FPSs FPS's +F.Q. FQ +fr FR +F. R. FR +F.R. FR +frs FRS +Fru FRU +Frwydr FRWYDR +fsf FSF +fs FS +F. s F's +Fs F's +F. S. FS +F.S. FS +fsn FSN +FSSs FSS's +FTAs FTA's +Ft. Fort +F. T. FT +F.T. FT +ftp FTP +FtsA FTSA +Ftuh FTUH +F. V. FV +F.V. FV +F&W F and W +F. W. FW +F.W. FW +fwr FWR +fx FX +Fyb FYB +F.Y. FY +Fyn FYN +fyi FYI +fyr FYR +Fyw FYW +F.Z. FZ +Gaac GAAC +G. A. GA +G.A. GA +Gbe GBE +GbE GBE +G. B. GB +G.B. GB +GB&NDR GB and NDR +G&C G and C +G. C. GC +G.C. GC +GCSEs GCSE's +GC&SF GC and SF +Gdal GDAL +Gde GDE +gd GD +G. D. GD +G.D. GD +Gdow GDOW +GDPs GDP's +gdr GDR +Gebr GEBR +Gebt GEBT +Ged GED +ge GE +G. E. GE +G.E. GE +Gek GEK +Gen. general +geq GEQ +Geu GEU +Gev GEV +GeV GEV +Gfa GFA +GF&A GFANDA +G. F. GF +G.F. GF +G. G. GG +G.G. GG +Ghe GHE +G. H. GH +G.H. GH +GHGs GHG's +Ghir GHIR +ghra GHRA +GH&SA GH and SA +Giei GIEI +G. I. GI +G.I. GI +G. J. GJ +G.J. GJ +G. K. GK +G.K. GK +Gla GLA +G&L G and L +G. L. GL +G.L. GL +gli GLI +Gli GLI +Glis GLI's +glnA GLNA +Glos GLO's +gma GMA +GmbH GMBH +Gmel GMEL +gm GM +G. M. GM +G.M. GM +gmin GMIN +GMOs GMO's +GMs GM's +gmt GMT +Gmul GMUL +gnb GNB +G. N. GN +G.N. GN +GNP&BR GNP and BR +gnp GNP +GNPs GNP's +G.O. GO +GPCRs GPCR's +GPdI GPDI +G&P G and P +gp GP +G. P. GP +G.P. GP +GpIIb GPIIB +GPMGs GPMG's +GPRs GPR's +GPUs GPU's +G. Q. GQ +grac GRAC +gra GRA +Gra GRA +Grbac GRBAC +GRBs GRB's +grc GRC +GRCs GRC's +gre GRE +Gre GRE +gr GR +G. R. GR +G.R. GR +grrl GRRL +Gryf GRYF +gry GRY +Gry GRY +grz GRZ +G&S G and S +Gs G's +G. S. GS +G.S. GS +GSIs GSI's +GSOp GSOP +GTAs GTA's +gt GT +G. T. GT +G.T. GT +GTi GTI +GTs GT's +G. U. GU +G.U. GU +GUIs GUI's +G. V. GV +G.V. GV +GvpA GVPA +GWe GWE +Gwet GWET +gw GW +G. W. GW +G.W. GW +Gwi GWI +gwr GWR +Gy GY +G. Y. GY +Gyn GYN +Gyps GYP's +gyu GYU +Gyu GYU +H. A. HA +H.A. HA +H&A H and A +HBCo HBCO +HBGAs HBGA's +H. B. HB +H.B. HB +H&BR H and BR +HCEs HCE's +H. C. HC +H.C. HC +hcl HCL +HClO HCLO +HCoV HCOV +HCPs HCP's +Hcy HCY +Hdad HDAD +hDAF HDAF +HDDs HDD's +hd HD +H. D. HD +H.D. HD +hdh HDH +HDi HDI +H. E. HE +H.E. HE +hezb HEZB +Hezb HEZB +H&F H and F +H. F. HF +H.F. HF +hgcA HGCA +hg HG +H. G. HG +H.G. HG +HgO HGO +HGPs HGP's +HgU HGU +HGVs HGV's +H&H H and H +H. H. HH +H.H. HH +H. I. HI +H.I. HI +Hizb HIZB +hja HJA +H. J. HJ +H.J. HJ +Hkam HKAM +H&K H and K +H. K. HK +H.K. HK +Hla HLA +H. L. HL +H.L. HL +Hluk HLUK +hlutr HLUTR +Hly HLY +Hman HMAN +H&M H and M +hm HM +H. M. HM +H.M. HM +HMOs HMO's +hMRE HMRE +H&M's H and M's +H. N. HN +H.N. HN +H. O. HO +H.O. HO +hpc HPC +hp HP +H. P. HP +H.P. HP +H.Q. HQ +H&R H and R +H. R. HR +H.R. HR +hroa HROA +HRo HRO +hrs HRS +HRs HR's +hscy HSCY +Hsee HSEE +hs HS +Hs H's +H. S. HS +H.S. HS +hsi HSI +hSlo HSLO +HSPs HSP's +HSTs HST's +hsv HSV +hTAS HTAS +HTAs HTA's +htc HTC +ht HT +H. T. HT +H.T. HT +hti HTI +Htin HTIN +Htoo HTOO +HtrA HTRA +HTRs HTR's +http HTTP +https HTTPS +Htwa HTWA +H. U. HU +HVCs HVC's +hvcv HVCV +hvem HVEM +Hvem HVEM +H. V. HV +H.V. HV +H. W. HW +H.W. HW +hwy HWY +Hwy HWY +hxt HXT +H. Y. HY +H.Y. HY +H. Z. HZ +HZ&PC HZ and PC +iaaf IAAF +IaaS IAAS +Iaca IACA +Iacob IACOB +IACTs IACT's +Iacub IACUB +iagt IAGT +I. A. IA +I.A. IA +Iapa IAPA +iar IAR +Iar IAR +IAUCs IAUC's +Iax IAX +Ibac IBAC +Ibaes IBAE's +IBCs IBC's +IBDA'A IBDAA +Ibda IBDA +ibdal IBDAL +I. B. IB +I.B. IB +Ibm IBM +Ibne IBNE +ibn IBN +Ibra IBRA +Ibsa IBSA +Ibs IB's +iBT IBT +Ibu IBU +iby IBY +ICBMs ICBM's +Icche ICCHE +icf ICF +I&C I and C +ic IC +Ic IC +I. C. IC +I.C. IC +Ici ICI +iCN ICN +Ico ICO +IcRn ICRN +ics ICS +ICs IC's +ICTs ICT's +ICTVdB ICTVDB +ICv ICV +iCyt ICYT +Iddaa IDDAA +Idd IDD +ID'd IDD +id ID +I. D. ID +I.D. ID +Idi IDI +Idir IDIR +IDLHs IDLH's +Idm IDM +IDPs IDP's +ids IDS +IDs ID's +IDx IDX +i.e. that is +Ifa IFA +ifc IFC +IfF IFF +Ifft IFFT +ifi IFI +Ifi IFI +IFIs IFI's +IfM IFM +Ifni IFNI +IFNs IFN's +Ifop IFOP +IFRSs IFRS's +Iga IGA +IgA IGA +IGFs IGF's +'ig IG +Ig IG +I.G. IG +igi IGI +IgM IGM +ign IGN +Ign IGN +Ih IH +I. H. IH +ihi IHI +ihm IHM +ihn IHN +Ihn IHN +Ihor IHOR +Iht IHT +IIb IIB +IIc IIC +IIfx IIFX +Iiga IIGA +IIga IIGA +IIgs IIG's +IIGs IIG's +IIIb IIIB +IIIBy IIIBY +IIIc IIIC +IIId IIID +I. I. II +I.I. II +ija IJA +Ija IJA +ij IJ +I. J. IJ +Ijok IJOK +Ijui IJUI +Ikh IKH +I.K. IK +IKr IKR +Iksa IKSA +Iku IKU +Ilbe ILBE +IL&FS IL and FS +ili ILI +Ili ILI +I. L. IL +I.L. IL +Ilm ILM +ilu ILU +Ilu ILU +IMbd IMBD +imc IMC +imdb IMDB +IMDb. IMDB +IMDb IMDB +ime IME +I&M I and M +I. M. IM +I.M. IM +Iml IML +Imm IMM +Imms' IMM's +imoa IMOA +impr IMPR +Impr IMPR +iMSNs IMSN's +Imst IMST +Inba INBA +inb INB +incl INCL +ind IND +Ind IND +Infs INF's +inHg INHG +I. N. IN +I.N. IN +InlB INLB +InP INP +Inre INRE +Inspx INSPX +Ints INT's +INTs INT's +Intu INTU +inv INV +Ioba IOBA +IODs IOD's +io. IO +io IO +.I.o. IO +Io. IO +Io IO +.IO IO +I/O IO +Ioka IOKA +Iok IOK +IoM IOM +IOPs IOP's +iOS IOS +Io's IO's +I/Os IO's +I.O.'s IO's +Iosu IOSU +IoT IOT +IoW IOW +IPCs IPC's +ip IP +Ip IP +I.P. IP +Ipo IPO +IPPs IPP's +Ippu IPPU +iPPV IPPV +iPSC IPSC +iPSCs IPSC's +ipse IPSE +iPS IPS +IPs IP's +IPv IPV +iQ IQ +I.Q. IQ +Irgm IRGM +Irig IRIG +Iril IRIL +i'r IR +ir IR +Ir IR +I. R. IR +I.R. IR +Irla IRLA +Irmis IRMI's +iro IRO +Iro IRO +Irra IRRA +IRs IR's +iru IRU +Iru IRU +ISAv ISAV +Isba ISBA +isbn ISBN +ISBNs ISBN's +isCf ISCF +iSC ISC +Isc ISC +Isd ISD +isdn ISDN +Isgec ISGEC +ishq ISHQ +Ishq ISHQ +I. S. IS +I.S. IS +Isl ISL +ISMNs ISMN's +ISPs ISP's +Isra'il ISRAIL +Isra ISRA +iss ISS +Iss IS's +ISSNs ISSN's +Isu ISU +Ite ITE +ITHs ITH's +iti ITI +Iti ITI +I.T. IT +itk ITK +Itk ITK +ITNNs ITNN's +Itoi ITOI +iTP ITP +Itse ITSE +itt ITT +Itu ITU +Itz ITZ +Iucn IUCN +IUDs IUD's +Iuz IUZ +I. V. IV +I.V. IV +IVs IV's +Ivu IVU +Iwas IWA's +Iwaz IWAZ +IWBs IWB's +iwi IWI +iw IW +I. W. IW +iwrg IWRG +iwspy IWSPY +Iwuh IWUH +IXb IXB +IXBs IXB's +IXCs IXC's +IXe IXE +iXL IXL +ixtle IXTLE +Iya IYA +Iyar IYAR +iyem IYEM +Iyi IYI +iy IY +I. Y. IY +I.Y. IY +Jaf JAF +J. A. JA +J&B J and B +jb JB +J. B. JB +J.B. JB +jcis JCIS +jc JC +J. C. JC +J.C. JC +JCRs JCR's +JdeBP JDEBP +jdi JDI +jd JD +J. D. JD +J.D. JD +jdk JDK +J&D's J and D's +J.D.s JD's +J. E. JE +J.E. JE +jf JF +J. F. JF +J.F. JF +J. G. JG +J.G. JG +jh JH +J. H. JH +J.H. JH +JHs JH's +JHSVs JHSV's +JHud JHUD +J. I. JI +J.I. JI +J&J JANDJ +J. J. JJ +J.J. JJ +Jka JKA +jkd JKD +J&K J and K +J. K. JK +J.K. JK +jkx JKX +J. L. JL +J.L. JL +J. M. JM +J.M. JM +jnb JNB +J. N. JN +J.N. JN +Jno JNO +jnr JNR +J. O. JO +J.O. JO +jpg JPG +J&P J and P +jp JP +J.P. JP +J.Q. JQ +J&R J and R +J. R. JR +J.R. JR +Jr. junior +jr. junior +Jr junior +jr junior +J.R.'s JR's +jsb JSB +Js J's +J. S. JS +J.S. JS +J&T J and T +J. T. JT +J.T. JT +JT&KW JT and KW +J. U. JU +J&V J and V +J. V. JV +J.V. JV +Jwa JWA +J. W. JW +J.W. JW +jx JX +J. Y. JY +J.Y. JY +Jym JYM +Jymn JYMN +J. Z. JZ +J.Z. JZ +K8S KUBERNETES +K. A. KA +K.A. KA +kbi KBI +K&B K and B +K. B. KB +K.B. KB +K. C. KC +K.C. KC +K.C.'s KC's +kDa KDA +kdal KDAL +K&D K and D +kd KD +K. D. KD +K.D. KD +K. D.'s KD's +K.D.'s KD's +ke KE +Ke KE +K. E. KE +K.E. KE +kfc KFC +K.F. KF +kgr KGR +kgt KGT +KgU KGU +kgv KGV +Khizr KHIZR +K. H. KH +K.H. KH +Khlav KHLAV +Khmu KHMU +Khri KHRI +Khru KHRU +K. I. KI +K.I. KI +Kjer KJER +kj KJ +K. J. KJ +K.J. KJ +Kjop KJOP +KJo's KJO's +K. K. KK +K.K. KK +Kle KLE +kl KL +K. L. KL +K.L. KL +K&M K and M +km KM +K. M. KM +K.M. KM +K&N K and N +kn KN +K. N. KN +K.N. KN +k'o KO +K. O. KO +K.O. KO +KPIs KPI's +K&P K and P +K. P. KP +K.P. KP +Kppen KPPEN +kptm KPTM +krc KRC +KRCs KRC's +K.R.I.T.'s KRIT's +Krka KRKA +kr KR +K. R. KR +K.R. KR +Krne KRNE +kroz KROZ +Kroz KROZ +Kru KRU +ksa KSA +kset KSET +Kseur KSEUR +Ksevt KSEVT +ks KS +K. S. KS +K.S. KS +Ktav KTAV +KT&K KT and K +kt KT +K. T. KT +K.T. KT +Kuaa KUAA +Kud KUD +kuih KUIH +K. U. KU +K.U. KU +Kutb KUTB +Kvik KVIK +Kvit KVIT +K. V. KV +K.V. KV +Kvyat KVYAT +kwa KWA +kwe KWE +Kyse KYSE +kyt KYT +Kyt KYT +kz KZ +K. Z. KZ +L. A. LA +L.A.'s LA's +Lay's LAY's +LBi LBI +L. B. LB +L.B. LB +LBPs LBP's +LCAs LCA's +lcc LCC +LCCs LCC's +LC&DR LC and DR +LCDs LCD's +LCIs LCI's +L. C. LC +L.C. LC +lcn LCN +LCSs LCS's +LCTs LCT's +LCVPs LCVP's +Lda LDA +LDCs LDC's +L. D. LD +L.D. LD +L. E. LE +L.E. LE +LFFCs LFFC's +L. F. LF +L.F. LF +LFs LF's +LFSRs LFSR's +L. G. LG +L.G. LG +LGMs LGM's +LG&RDD LG and RDD +LGs LG's +LGUs LGU's +LGVs LGV's +Lha LHA +L. H. LH +L.H. LH +L. I. LI +L.I. LI +Lje LJE +L. J. LJ +L.J. LJ +L. K. LK +lks LKS +Llapi LLAPI +lle LLE +lli LLI +L&L L and L +L. L. LL +L.L. LL +llp LLP +llu LLU +LMGs LMG's +lm LM +L. M. LM +L.M. LM +LMPs LMP's +LMQs LMQ's +LnAIB LNAIB +L&N L and N +L. N. LN +L.N. LN +L.O.C.'s LOC's +Lokk LOKK +l'OL LOL +L. O. LO +L.O. LO +LPARs LPAR's +L&P L and P +lp LP +L. P. LP +L.P. LP +L&PM L and PM +LPMud LPMUD +LPs LP's +LPThe LPTHE +lr LR +L. R. LR +L.R. LR +LRTs LRT's +LRVs LRV's +LSDs LSD's +LSi LSI +ls LS +Ls L's +L. S. LS +L.S. LS +LSTs LST's +LSVCCs LSVCC's +Ltda LTDA +Ltd. limited +L&T L and T +Lt. lieutenant +lt LT +L. T. LT +L.T. LT +ltoh LTOH +L. U. LU +lv LV +L. V. LV +L.V. LV +L. W. LW +L.W. LW +L.Y. LY +L&YR L and YR +M. A. MA +M.A. MA +MAX-Q MAXQ +M&A M and A +Mbewu MBEWU +mbi MBI +M. B. MB +M.B. MB +mBo MBO +Mbre MBRE +mbr MBR +MCCs MCC's +M. C. MC +M.C. MC +MCs MC's +mcyG MCYG +mdb MDB +MDCs MDC's +mdDA MDDA +MD&DI MD and DI +Mde MDE +MDGs MDG's +MDHUs MDHU's +mDia MDIA +MDic MDIC +mdla MDLA +md MD +M. D. MD +M.D. MD +mdr MDR +M. E. ME +M.E. ME +MFDs MFD's +MFe MFE +M. F. MF +M.F. MF +MFTs MFT's +Mgadla MGADLA +Mgal MGAL +mga MGA +Mga MGA +Mgbo MGBO +MGen MGEN +M&G M and G +M. G. MG +M.G. MG +M&GN M and GN +M&GR's M and GR's +Mha MHA +MHCs MHC's +MH&L MH and L +M&H M and H +M. H. MH +M.H. MH +Mhor MHOR +Mhow MHOW +M.I.A.'s MIA's +M&I M and I +M. I. MI +M.I. MI +M&J M and J +M. J. MJ +M.J. MJ +M. K. MK +M.K. MK +MKs MK's +MLAs MLA's +mlc MLC +MLCs MLC's +mlg MLG +mli MLI +Mlle MLLE +M&L M and L +M. L. ML +M.L. ML +MLPs MLP's +MM&A MMANDA +M&M M and M +M. M. MM +M.M. MM +M&M's M and M's +MNCs MNC's +M&NF M and NF +M. N. MN +M.N. MN +M.O.G.U.E.R.A.'s MOGUERA's +M. O. MO +M.O. MO +MpA MPA +MPAs MPA's +MPBu MPBU +MP&I MP and I +MPi MPI +MPLMs MPLM's +M&P M and P +M. P. MP +M.P. MP +MPPs MPP's +M.P.'s MP's +MPs MP's +MPThe MPTHE +Mpu MPU +M. Q. MQ +Mra MRA +MRBs MRB's +mre MRE +MR&LE MR and LE +Mr. mister +M. R. MR +M.R. MR +mRNA MRNA +mRNAs MRNA's +Mrs. misses +MRTs MRT's +msd MSD +Mse MSE +M&S M and S +Ms. miss +M. S. MS +M.S. MS +MSPs MSP's +mst MST +Mta MTA +MTAs MTA's +M&T M and T +M. T. MT +M.T. MT +Mtor MTOR +mtvU MTVU +MTVu MTVU +M. U. MU +M.U.s MU's +MVMs MVM's +M. V. MV +M.V. MV +MVPs MVP's +MVs MV's +MWe MWE +M. W. MW +M.W. MW +M. X. MX +myb MYB +myc MYC +Myc MYC +MyDD MYDD +M. Y. MY +M.Y. MY +MySQL MYSQL +M. Z. MZ +N. A. NA +N.A. NA +N.B.A.'s NBA's +NBAs NBA's +N. B. NB +N.B. NB +nbs NBS +NBTwo NBTWO +NCAAs NCAA's +NCCs NCC's +N. C. NC +N.C. NC +NCOs NCO's +N. D. ND +N.D. ND +N. E. NE +N.E. NE +ner NER +NFATc NFATC +nfed NFED +N. F. NF +N.F. NF +Nge NGE +N. G. NG +N.G. NG +NGOs NGO's +Nha NHA +NHCEs NHCE's +NHCs NHC's +nhi NHI +NHM&W NHM and W +N. H. NH +N.H. NH +NHPs NHP's +N. I. NI +N.I. NI +N'I NI +N. J. NJ +N.J. NJ +Njoo NJOO +N.J.'s NJ's +N. K. NK +N.K. NK +nkvd NKVD +nkv NKV +N. L. NL +N.L. NL +NLRs NLR's +N. M. NM +N.M. NM +N. N. NN +N.N. NN +NPCs NPC's +NPMs NPM's +N. P. NP +N.P. NP +NPs NP's +N. Q. NQ +nri NRI +Nri NRI +NRIs NRI's +NRJs NRJ's +nr NR +N. R. NR +N.R. NR +N. S. NS +N.S. NS +NSOs NSO's +Nta NTA +ntb NTB +N. T. NT +N.T. NT +ntw NTW +N.U. NU +NvDA's NVDA's +N. V. NV +N.V. NV +nwa NWA +N&W N and W +N. W. NW +N.W. NW +nyc NYC +N. Y. NY +N.Y. NY +NX-SOC NXSOC +N'Zif NZIF +NZiK NZIK +N'Zi NZI +N.Z. NZ +O. A. OA +O.A. OA +obl OBL +Oblt OBLT +O.B. OB +O. C. OC +O.C. OC +Octl OCTL +O. D. OD +O.E. OE +O.F. OF +ofr OFR +O&G O and G +O. G. OG +O.G. OG +O. H. OH +O.H. OH +O. I. OI +O.I. OI +O. J. OJ +O.J. OJ +OK'd OKD +OKd OKD +oke OKE +Oke OKE +O&K O and K +O. K. OK +O.K. OK +O. L. OL +O.L. OL +Olo OLO +Olov OLOV +ols OLS +OLs OL's +olvwm OLVWM +olwm OLWM +O&MFL O and MFL +omg OMG +OMGs OMG's +O. M. OM +O.M. OM +OmOm OMOM +Om's OM's +oncu ONCU +ond OND +onf ONF +O. N. ON +O.N. ON +O&O O and O +O. O. OO +O.O. OO +Oop OOP +oor OOR +oose OOSE +Oo's OO's +Ootw OOTW +OPMs OPM's +O. P. OP +O.P. OP +opr OPR +Opr OPR +ORFs ORF's +OR&N OR and N +O. R. OR +osaa OSAA +osa OSA +Osa OSA +OSBs OSB's +O. S. OS +O.S. OS +OSTs OST's +osv OSV +O.T. OT +OTs OT's +Otu OTU +otv OTV +O.U. OU +ovca OVCA +Ovca OVCA +Ovda OVDA +Ovo OVO +ov OV +Ov OV +O. V. OV +O.V. OV +OWCs OWC's +O. W. OW +O.W. OW +pa PA +P. A. PA +P.A. PA +pBCE PBCE +pbc PBC +pb PB +P. B. PB +P.B. PB +PCBs PCB's +PCDDs PCDD's +PCeU PCEU +PCIe PCIE +pci PCI +pcl PCL +PcoA PCOA +pc PC +P. C. PC +P.C. PC +PCRev PCREV +pcs PCS +P.C.s PC's +pDAB PDAB +PDAs PDA's +pdbp PDBP +PDBsum PDBSUM +PDCs PDC's +PD&D PD and D +PDEs PDE's +pdf PDF +PDFs PDF's +P&D P and D +P. D. PD +P.D. PD +PDPs PDP's +PDs PD's +P.E.I.'s PEI's +P. E. PE +P.E. PE +PFs PF's +PFW&C PFW and C +PGMs PGM's +PG&N PG and N +P&G P and G +P. G. PG +P.G. PG +Phlo PHLO +P&H P and H +P. H. PH +P.H. PH +php PHP +PHPs PHP's +P&I P and I +P. I. PI +P.I. PI +Piz PIZ +pj PJ +P. J. PJ +P.J. PJ +PJs PJ's +P. K. PK +P.K. PK +plc PLC +PLCs PLC's +P. L. PL +P.L. PL +PMMoV PMMOV +p.m. PM +p.m PM +P. M. PM +P.M. PM +PMs PM's +PNaCl PNACL +pna PNA +Pnau PNAU +pnb PNB +PNEs PNE's +PNETs PNET's +pneus PNEUS +png PNG +PNNs PNN's +PNoy PNOY +pn PN +P. N. PN +P.N. PN +P.O.D.'s POD's +P&O P and O +P. O. PO +P.O. PO +P.O.W.'s POW's +P&PH P and PH +PPi PPI +P. P. PP +P.P. PP +PPVs PPV's +PPy PPY +pr PR +P. R. PR +P.R. PR +P. s P's +P.'s P's +Ps P's +P. S. PS +P.S. PS +PSSAs PSSA's +pTA PTA +PTAs PTA's +Pte PTE +PTEs PTE's +ptf PTF +Ptie PTIE +PTLs PTL's +Ptol PTOL +pt PT +P. T. PT +P.T. PT +PTTs PTT's +P'Twa PTWA +Puiu PUIU +Pul PUL +PVAs PVA's +PvdA PVDA +pve PVE +P. V. PV +P.V. PV +pvr PVR +PVs PV's +pwll PWLL +P&W P and W +P. W. PW +P.W. PW +Pyi PYI +Pyk PYK +Pyl PYL +PyL PYL +P. Y. PY +Pyu PYU +Pyw PYW +Pyx PYX +P. Z. PZ +Q. A. QA +Q&A QANDA +Q&A's QANDA's +Q&As QANDAS +QbA QBA +Q.B. QB +Q.C. QC +Q. E. QE +Q.H. QH +Q.I. QI +Q. J. QJ +Q. N. QN +QPOs QPO's +Q.V. QV +R. A. RA +R.A. RA +R&A R and A +R&AW R and AW +RbAg RBAG +RBCs RBC's +RBIs RBI's +R&B R and B +R. B. RB +R.B. RB +rca RCA +R&C R and C +rc RC +R. C. RC +R.C. RC +RCTs RCT's +RdE RDE +R&D R and D +R. D. RD +R.D. RD +R.E.M.'s REM's +REPLs REPL's +R. E. RE +R.E. RE +R. F. RF +R.F. RF +R&G R and G +R. G. RG +R.G. RG +RHIBs RHIB's +Rhiw RHIW +rhl RHL +RhoG RHOG +Rho's RHO's +R. H. RH +R.H. RH +R&I R and I +R. I. RI +R.I. RI +riu RIU +Riu RIU +rivs RIVS +Rivu RIVU +Rixt RIXT +rjf RJF +R. J. RJ +R.J. RJ +R. K. RK +R.K. RK +R. L. RL +R.L. RL +rly RLY +RMDs RMD's +RMLs RML's +rm RM +R. M. RM +R.M. RM +RMSDs RMSD's +RMs RM's +rna RNA +RNAs RNA's +Rnet RNET +Rnic RNIC +RNNs RNN's +RNPs RNP's +R. N. RN +R.N. RN +rOmpB ROMPB +Rooi ROOI +R. O. RO +R.O. RO +ROVs ROV's +Roxb ROXB +Roxx ROXX +RP&C RP and C +rpc RPC +Rpe RPE +rpg RPG +RPGs RPG's +rpm RPM +R. P. RP +R.P. RP +RPs RP's +R. Q. RQ +R.Q. RQ +rra RRA +rrd RRD +Rreli RRELI +rre RRE +rRNA RRNA +rRNAs RRNA's +R&R R and R +rr RR +R. R. RR +R.R. RR +rsh RSH +Rsis RSI's +rsly RSLY +RsmA RSMA +RSpec RSPEC +rs RS +Rs R's +R. S. RS +R.S. RS +RSu RSU +rtb RTB +RTCs RTC's +RTEjr RTEJR +RTeOR RTEOR +RTOs RTO's +rtPA RTPA +rt RT +R. T. RT +R.T. RT +rts RTS +RTVFBiH RTVFBIH +Ruao RUAO +Rukn RUKN +Ruk RUK +Rupf RUPF +Rupr RUPR +rup RUP +Rup RUP +R. U. RU +R.U. RU +RUN:AI RUN AI +RutB RUTB +Ruu RUU +Ruwa RUWA +Ruy RUY +Ruyt RUYT +Rvat RVAT +Rvo RVO +R. V. RV +R.V. RV +rvs RVS +rwa RWA +Rwa RWA +rwb RWB +RWEs RWE's +rwjf RWJF +R. W. RW +R.W. RW +Rxa RXA +rz RZ +R. Z. RZ +Sa'id SAID +S. A. SA +S.A. SA +Saxl SAXL +Sa'yo SAYO +Sbai SBAI +sbc SBC +SBCs SBC's +sbk SBK +Sborz SBORZ +sb SB +S. B. SB +S.B. SB +Sbu SBU +scr SCR +SCRs SCR's +scry SCRY +sc SC +S. C. SC +S.C. SC +Scuf SCUF +Scymn SCYMN +SD&AE SD and AE +Sdei SDEI +sde SDE +Sde SDE +SDG&E SDG and E +sdk SDK +S. D. SD +S.D. SD +SDSM&T's SDSM and T's +Sejms SEJM's +sejr SEJR +Sekl SEKL +Sek SEK +S. E. SE +S.E. SE +ses SES +Ses SE's +SEs SE's +S.E.S.'s SES's +Seyh SEYH +sfadb SFADB +Sfax SFAX +SFFCo SFFCO +sfn SFN +'sf SF +sf SF +S. F. SF +S.F. SF +sfs SFS +S.F.'s SF's +SG&A SGANDA +sgb SGB +Sgip SGIP +sgml SGML +sgra SGRA +S. G. SG +S.G. SG +SGSNs SGSN's +Sgt. sergeant +sgt SGT +S.H.I.E.L.D.'s SHIELD's +Shma SHMA +S. H. SH +S.H. SH +S. I. SI +S.I. SI +sjef SJEF +Sjon SJON +S. J. SJ +S.J. SJ +S. K. SK +S.K. SK +S.L.A.A.'s SLAA's +sla SLA +SLAs SLA's +S&L S and L +SLS&E SLS and E +S. L. SL +S.L. SL +SMe SME +SMEs SME's +SmI SMI +SMPSs SMPS's +smr SMR +S&M S and M +sm SM +S. M. SM +S.M. SM +S.M.'s SM's +SMs SMS +Smyl SMYL +SNESjr SNESJR +sngle SNGLE +SNPs SNP's +SnSe SNSE +S. N. SN +S.N. SN +S. O. SO +S.O. SO +Sos SO's +Sovn SOVN +Sov SOV +S&P 500 S and P five hundred +SPCAs SPCA's +Spe SPE +Spoa SPOA +SpPIn SPPIN +S&P S and P +S. P. SP +S.P. SP +SPs SP's +SPUs SPU's +sql SQL +sq SQ +S. Q. SQ +Sra SRA +SRAs SRA's +Srba SRBA +Srbi SRBI +SRBs SRB's +SRGs SRG's +Srhir SRHIR +SRLGs SRLG's +SRMs SRM's +Srni SRNI +srp SRP +sr SR +S. R. SR +S.R. SR +Srul SRUL +srx SRX +SSAs SSA's +SSDs SSD's +ssl SSL +SSoSV SSOSV +SSRIs SSRI's +S&S S and S +ss SS +S. s S's +Ss S's +S. S. SS +S.S. SS +Ssu SSU +ssw SSW +Ssy SSY +ST&AJ ST and AJ +STDs STD's +stfv STFV +STG-TWO STGTWO +STGs STG's +STIs STI's +STi STI +stl STL +Stryj STRYJ +S&T S and T +STScI STSCI +S. T. ST +S.T. ST +suo SUO +Suo SUO +S. U. SU +Susz SUSZ +SUTs SUT's +Suu SUU +SUVs SUV's +Suy SUY +Svac SVAC +svar SVAR +sve SVE +Sve SVE +SVMs SVM's +svn SVN +svom SVOM +sv SV +S. V. SV +S.V. SV +SVTs SVT's +Swe SWE +SwRI SWRI +S&W S and W +sw SW +S. W. SW +S.W. SW +sx SX +S.X. SX +SysML SYSML +SysRq SYSRQ +Sys SY's +SysV SYSV +S. Y. SY +S.Y. SY +SyT SYT +syv SYV +Syxx SYXX +Szasz SZASZ +Szer SZER +szkic SZKIC +Szklo SZKLO +Szlak SZLAK +SzMME SZMME +Szpir SZPIR +sz SZ +S. Z. SZ +Szu SZU +Szyk SZYK +taf TAF +Taf TAF +T. A. TA +T.A. TA +T&A T and A +TBCs TBC's +TBMs TBM's +tbh TBH +T. B. TB +T.B. TB +TCiAP TCIAP +TCKs TCK's +tc TC +T. C. TC +T.C. TC +TCUs TCU's +tcu TCU +tdb TDB +TDCi TDCI +TdIF TDIF +TDs TD's +T. D. TD +T.D. TD +Teatr TEATR +T. E. TE +T.E. TE +T&F T and F +tf TF +T. F. TF +T.F. TF +tge TGE +tgf TGF +tg TG +T. G. TG +T.G. TG +TGVs TGV's +tgv TGV +Thok THOK +ThSe THSE +T. H. TH +T.H. TH +T.I.'s TI's +T.I. TI +Tiu TIU +tiv TIV +Tiv TIV +Tiy's TIY's +T. J. TJ +T.J. TJ +t'ju TJU +tjz TJZ +tko TKO +Tko TKO +tk TK +T. K. TK +T.K. TK +Tlas TLA's +TlCu TLCU +TLDs TLD's +tli TLI +tlp TLP +TLRs TLR's +TLs TL's +tl TL +T. L. TL +T.L. TL +Tluk TLUK +Tluszcz TLUSZCZ +tmc TMC +TMGs TMG's +tmos TMOS +tmRNA TMRNA +TMSs TMS's +T.M.s TM's +tm TM +T. M. TM +T.M. TM +tna TNA +tnbc TNBC +TNTAs TNTA's +tn TN +T. N. TN +T.N. TN +T.O.'s TO's +T. O. TO +T.O. TO +tou TOU +Tou TOU +ToU TOU +tPA TPA +tpr TPR +tp TP +T. P. TP +T.P. TP +TPVs TPV's +T.Q. TQ +tra TRA +Tra TRA +Trbic TRBIC +Trcek TRCEK +TrkA TRKA +Trmcic TRMCIC +tRNA TRNA +TRPs TRP's +trs TRS +tr TR +T. R. TR +T.R. TR +truTV TRUTV +TruTV TRUTV +Trve TRVE +Tsa TSA +tsit TSIT +Tsiv TSIV +tso TSO +Tso TSO +tsr TSR +Ts T's +T. S. TS +T.S. TS +tsus TSUS +Tsvi TSVI +Tta TTA +tteok TTEOK +tte TTE +ttp TTP +T&T T and T +T. T. TT +T.T. TT +T'uqu TUQU +T.U. TU +Tuzk TUZK +tvaan TVAAN +Tvam TVAM +tva TVA +TVETs TVET's +TVii TVII +TVIn TVIN +TVi TVI +tvo TVO +Tvo TVO +TVRi TVRI +tvr TVR +TVSpy TVSPY +tvs TVS +TVs TV's +tVTA TVTA +tv TV +T. V. TV +T.V. TV +Twa TWA +Twi TWI +TWTs TWT's +T. W. TW +T.W. TW +TxDOT TXDOT +T.X. TX +tya TYA +Tyk TYK +Tza'ar TZAAR +Tze TZE +Tzrif TZRIF +Tzuh TZUH +Tzvi TZVI +Ua's UA's +UAs UA's +uat UAT +U.A. UA +UAVs UAV's +Uba UBA +UbcM UBCM +Ube UBE +ubi UBI +Ubi UBI +Ucar UCAR +Uca UCA +Ucmak UCMAK +Ucn UCN +uCs UC's +Uc UC +U.C. UC +Ucuncu UCUNCU +Uczta UCZTA +Uda UDA +Udit UDIT +Udny UDNY +UDTs UDT's +ud UD +Ud UD +U. D. UD +udu UDU +Udu UDU +Ueda's UEDA's +Uéda UE acute DA +U. E. UE +UFOs UFO's +Ufot UFOT +ufo UFO +Ufo UFO +U. F. UF +Uga UGA +Uge UGE +Ugni UGNI +Ugra UGRA +Ugrszke UGRSZKE +Ug UG +Uhha UHHA +uhn UHN +UHTCs UHTC's +Uhud UHUD +U.H. UH +uhur UHUR +Uiam UIAM +Uibh UIBH +Uible UIBLE +Uig UIG +uisae UISAE +UiS UIS +UiTM UITM +uit UIT +Uit UIT +Uiy UIY +Ujed UJED +Ujsag UJSAG +uj UJ +U.J. UJ +Uka UKA +Ukhra UKHRA +Ukic UKIC +Uki UKI +Ukoh UKOH +Uko UKO +ukr UKR +Ukr UKR +Ukui UKUI +u'k UK +uk UK +Uk UK +U.K. UK +Ukwu UKWU +Ulic ULIC +Ull ULL +Ulms ULM's +UlSU ULSU +Uluj ULUJ +U.L. UL +ulus ULUS +Ulus ULU's +Ulwa ULWA +Ulwe ULWE +Umbr UMBR +umelcu UMELCU +UMe UME +Umla UMLA +Umme UMME +Umno UMNO +Umri UMRI +U. M. UM +U.M. UM +Unli UNLI +Unlu UNLU +unm UNM +unnd UNND +UNSh UNSH +Unst UNST +Uns UN's +U.N.'s UN's +Unt UNT +U.N. UN +Unz UNZ +UOCl UOCL +UofM UOFM +Uoho UOHO +UoMs UOM's +uORF UORF +UoW UOW +UPBs UPB's +Upd UPD +Upf UPF +Upir UPIR +UPnP UPNP +Uppu UPPU +U.P. UP +UPyD UPYD +uq UQ +Urbz URBZ +URCs URC's +Urdd URDD +Urei UREI +urf URF +Urla URLA +URLhttp URLHTTP +URLs URL's +Usal USAL +usan USAN +Usan USAN +usao USAO +usata USATA +usa USA +.USA USA +USAya USAYA +USBs USB's +Uscie USCIE +U.S.C.'s USC's +USDoE USDOE +usd USD +USFbA USFBA +USF&WS USF and WS +UShs USH's +Usia USIA +usih USIH +Uslu USLU +Usmar USMAR +Usna USNA +Usnic USNIC +Usoi USOI +Usos USO's +Uso USO +USPs USP's +U.S.'s US's +Usti USTI +Ustka USTKA +Usui USUI +usum USUM +U. S. US +U.S. US +Usut USUT +Usvit USVIT +Uta's UTA's +uta UTA +Uta UTA +Utca UTCA +Utd's UTD's +Utd UTD +utea UTEA +Utes UTE's +Uth UTH +uti UTI +Uti UTI +Utne UTNE +Utnur UTNUR +Uto UTO +utrci UTRCI +utsav UTSAV +Utsav UTSAV +Utsu UTSU +U.T. UT +Utu UTU +Utva UTVA +Uuh UUH +Uul UUL +Uulu UULU +Uusi UUSI +UUs UU's +Uuto UUTO +Uvac UVAC +Uvea UVEA +uvnitr UVNITR +Uvo UVO +U.V. UV +Uwasa UWASA +UWFi UWFI +V. A. VA +V.A. VA +V&A V and A +Vav VAV +Vay VAY +vb VB +V. B. VB +V.B. VB +VCDs VCD's +VCRs VCR's +VCSELs VCSEL's +VCs VC's +VCTs VCT's +vc VC +V. C. VC +V.C. VC +vda VDA +Vda VDA +VDCs VDC's +vdiq VDIQ +vdm VDM +V. D. VD +V.D. VD +vez VEZ +Vez VEZ +vfp VFP +vfr VFR +V. F. VF +V.F. VF +V'Ger VGER +vgmdb VGMDB +VGo VGO +VGSCs VGSC's +VGSoM VGSOM +V. G. VG +V.G. VG +V. H. VH +V.I.C.'s VIC's +Vict VICT +viita VIITA +vijf VIJF +vij VIJ +Vij VIJ +V.I.P.s VIP's +VIPs VIP's +V. I. VI +V.I. VI +VJs VJ's +V. J. VJ +V.J. VJ +V. K. VK +V.K. VK +Vlah VLAH +VLCCs VLCC's +vlei VLEI +Vlijt VLIJT +V. L. VL +V.L. VL +VMAs VMA's +vm VM +V. M. VM +V.M. VM +vner VNER +V. N. VN +V.N. VN +V.O. VO +Vov VOV +Voz VOZ +vpis VPIS +VPNs VPN's +vpu VPU +V. P. VP +V.P. VP +vq VQ +vrak VRAK +Vrba VRBA +Vrbuv VRBUV +Vrej VREJ +vrem VREM +Vrin VRIN +vrj VRJ +vrn VRN +vroee VROEE +vrou VROU +vrouw VROUW +Vrouw VROUW +Vrsac VRSAC +Vrtis VRTI's +V&R V and R +vr VR +V. R. VR +V.R. VR +VSANs VSAN's +VSATs VSAT's +Vsekh VSEKH +vse VSE +vso VSO +VSPs VSP's +vs. versus +_vs._ versus +vsyo VSYO +VTE VT eL +Vtic VTIC +VTi VTI +V&T's V and T's +V&T V and T +V.T. VT +Vuur VUUR +VVIPs VVIP's +V. V. VV +V.V. VV +VWs VW's +Vyg VYG +vyr VYR +vy VY +vz VZ +V. Z. VZ +Waay WAAY +Wa'il WAIL +Wakf WAKF +wa'l WAL +waqf WAQF +waqt WAQT +Waqt WAQT +Wasl WASL +Watfa WATFA +wau WAU +Wau WAU +W. A. WA +W.A. WA +waw WAW +waza WAZA +Waza WAZA +WBs WB's +W. B. WB +W.B. WB +W&CBR W and CBR +wc WC +W. C. WC +W.C. WC +W. D. WD +W.D. WD +W. E. WE +W.E. WE +WF&NW WF and NW +W&F W and F +W. F. WF +W.F. WF +wga WGA +wgn WGN +wg WG +W. G. WG +W.G. WG +W&H W and H +W. H. WH +W.H. WH +Whyld WHYLD +Wica WICA +wici WICI +Wif WIF +Wijk WIJK +Wiwa WIWA +W. I. WI +wjaz WJAZ +W&J's W and J's +W&J W and J +W. J. WJ +W.J. WJ +W. K. WK +W.K. WK +W&LE W and LE +Wley WLEY +wlrs WLRS +W. L. WL +W.L. WL +wml WML +W&M W and M +W. M. WM +W.M. WM +W. N. WN +W.N. WN +wnzaa WNZAA +W&OD W and OD +Worh WORH +W. O. WO +W.O. WO +wpt WPT +wp WP +W. P. WP +W.P. WP +Wrec WREC +W. R. WR +W.R. WR +ws WS +W. S. WS +W.S. WS +WTs WT's +WTTs WTT's +W. T. WT +W.T. WT +WUAs WUA's +Wudl WUDL +Wuhr WUHR +Wuhu WUHU +Wu's WU's +wuv WUV +W. U. WU +Wuz WUZ +WVa WVA +W. V. WV +W.V. WV +Wwe WWE +W. W. WW +W.W. WW +wx WX +W.Y. WY +wyzc WYZC +W. Z. WZ +xbg XBG +Xbra XBRA +xCo XCO +X. C. XC +xda XDA +XDRs XDR's +X. D. XD +Xfce XFCE +xf XF +X.F. XF +xh XH +X. H. XH +xk XK +xmc XMC +xml XML +X.M. XM +xO XO +X.O. XO +XPe XPE +X.Q. XQ +XSi XSI +xsr XSR +XTwas XTWA's +xt XT +X. W. XW +xy XY +xyz XYZ +xyZ XYZ +Y. A. YA +Y.A. YA +Y. B. YB +Y.B. YB +ycia YCIA +ycie YCIE +yc YC +Y. C. YC +Y.C. YC +Ydby YDBY +Yde YDE +YdiB YDIB +yd YD +Y. D. YD +Y.D. YD +Y.E. YE +YFCs YFC's +yfle YFLE +YF&R YF and R +yg YG +Y. G. YG +Y.G. YG +Y. H. YH +Y. I. YI +Y.I. YI +yj YJ +Y. J. YJ +Ykt YKT +Y. K. YK +Y.K. YK +Yle YLE +yl YL +Y. L. YL +Ymke YMKE +ym YM +Y. M. YM +yndi YNDI +Yndi YNDI +yne YNE +Yngve YNGVE +Ynis YNI's +yn YN +Yn YN +Y. N. YN +Y.N. YN +Y.O. YO +ypa YPA +Y.P. YP +Y. Q. YQ +yra YRA +Yra YRA +Y&R Y and R +YSK'da YSKDA +YSK'ya YSKYA +Ys Y's +Y.S. YS +yta YTA +Ytre YTRE +Y&T Y and T +Y. T. YT +Y.T. YT +Y. V. YV +Y. W. YW +Y.Y. YY +Z. A. ZA +Z.A. ZA +Zdar ZDAR +zda ZDA +zg ZG +Z.G. ZG +zh ZH +Z. H. ZH +Z.H. ZH +zijn ZIJN +zij ZIJ +Zij ZIJ +Z. I. ZI +Z. J. ZJ +Z.K. ZK +Z.L. ZL +Z.M. ZM +ZnO ZNO +Zpav ZPAV +Z.P. ZP +ZrI ZRI +Zsuzsa ZSUZSA +Z.W. ZW +Z.X. ZX +Z. Y. ZY +Z.Y. ZY +Z. Z. ZZ +Z.Z. ZZ +2.5G two point five g +7-eleven seven eleven +AAA triple a +C# c sharp +DeepMind deep mind +DeepStream deep stream +DevOps dev ops +DisplayPort display port +HuggingFace hugging face +ServiceNow service now +SuperCloud super cloud +Wi-Fi wifi +7-Eleven seven eleven +Deepmind deep mind +Deepstream deep stream +Devops dev ops +Displayport display port +Huggingface hugging face +Servicenow service now +Supercloud super cloud +wi-fi wifi +deepmind deep mind +deepstream deep stream +devops dev ops +displayport display port +huggingface hugging face +servicenow service now +supercloud super cloud +wi-fi wifi +3-D three d +401k four oh one k +401(k) four oh one k +401 (k) four oh one k \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cd6ff3921b32969add3b854b6a8339ff197cdf62 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/graph_utils.py @@ -0,0 +1,337 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py new file mode 100644 index 0000000000000000000000000000000000000000..a75da75085e7d3683082768c532bf80821a33866 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, insert_space + + +class AbbreviationFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. "ABC" -> tokens { abbreviation { value: "A B C" } } + + Args: + whitelist: whitelist FST + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, whitelist: 'pynini.FstLike', deterministic: bool = True): + super().__init__(name="abbreviation", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + # A.B.C. -> A. B. C. + graph = NEMO_UPPER + dot + pynini.closure(insert_space + NEMO_UPPER + dot, 1) + # A.B.C. -> A.B.C. + graph |= NEMO_UPPER + dot + pynini.closure(NEMO_UPPER + dot, 1) + # ABC -> A B C + graph |= NEMO_UPPER + pynini.closure(insert_space + NEMO_UPPER, 1) + + # exclude words that are included in the whitelist + graph = pynini.compose( + pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph + ) + + graph = pynutil.insert("value: \"") + graph.optimize() + pynutil.insert("\"") + graph = self.add_tokens(graph) + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..f19b75dce9b4f1fd3ca241661413c7a9871b5433 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py @@ -0,0 +1,151 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.date import get_four_digit_year_graph +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -23 -> cardinal { negative: "true" integer: "twenty three" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + self.lm = lm + self.deterministic = deterministic + # TODO replace to have "oh" as a default for "0" + graph = pynini.Far(get_abs_path("data/number/cardinal_number_name.far")).get_fst() + graph_au = pynini.Far(get_abs_path("data/number/cardinal_number_name_au.far")).get_fst() + self.graph_hundred_component_at_least_one_none_zero_digit = ( + pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) + ) @ graph + + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + single_digits_graph = pynini.invert(graph_digit | graph_zero) + self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) + + if not deterministic: + # for a single token allow only the same normalization + # "007" -> {"oh oh seven", "zero zero seven"} not {"oh zero seven"} + single_digits_graph_zero = pynini.invert(graph_digit | graph_zero) + single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross("0", "oh") + + self.single_digits_graph = single_digits_graph_zero + pynini.closure( + insert_space + single_digits_graph_zero + ) + self.single_digits_graph |= single_digits_graph_oh + pynini.closure(insert_space + single_digits_graph_oh) + + single_digits_graph_with_commas = pynini.closure( + self.single_digits_graph + insert_space, 1, 3 + ) + pynini.closure( + pynutil.delete(",") + + single_digits_graph + + insert_space + + single_digits_graph + + insert_space + + single_digits_graph, + 1, + ) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph = ( + pynini.closure(NEMO_DIGIT, 1, 3) + + (pynini.closure(pynutil.delete(",") + NEMO_DIGIT**3) | pynini.closure(NEMO_DIGIT**3)) + ) @ graph + + self.graph = graph + self.graph_with_and = self.add_optional_and(graph) + + if deterministic: + long_numbers = pynini.compose(NEMO_DIGIT ** (5, ...), self.single_digits_graph).optimize() + self.long_numbers = plurals._priority_union(long_numbers, self.graph_with_and, NEMO_SIGMA).optimize() + cardinal_with_leading_zeros = pynini.compose( + pynini.accep("0") + pynini.closure(NEMO_DIGIT), self.single_digits_graph + ) + final_graph = self.long_numbers | cardinal_with_leading_zeros + final_graph |= self.add_optional_and(graph_au) + else: + leading_zeros = pynini.compose(pynini.closure(pynini.accep("0"), 1), self.single_digits_graph) + cardinal_with_leading_zeros = ( + leading_zeros + pynutil.insert(" ") + pynini.compose(pynini.closure(NEMO_DIGIT), self.graph_with_and) + ) + self.long_numbers = self.graph_with_and | pynutil.add_weight(self.single_digits_graph, 0.0001) + # add small weight to non-default graphs to make sure the deterministic option is listed first + final_graph = ( + self.long_numbers + | get_four_digit_year_graph() # allows e.g. 4567 be pronounced as forty five sixty seven + | pynutil.add_weight(single_digits_graph_with_commas, 0.0001) + | cardinal_with_leading_zeros + ).optimize() + + one_to_a_replacement_graph = ( + pynini.cross("one hundred", "a hundred") + | pynini.cross("one thousand", "thousand") + | pynini.cross("one million", "a million") + ) + final_graph |= pynini.compose(final_graph, one_to_a_replacement_graph.optimize() + NEMO_SIGMA).optimize() + # remove commas for 4 digits numbers + four_digit_comma_graph = (NEMO_DIGIT - "0") + pynutil.delete(",") + NEMO_DIGIT**3 + final_graph |= pynini.compose(four_digit_comma_graph.optimize(), final_graph).optimize() + + self.final_graph = final_graph + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def add_optional_and(self, graph): + graph_with_and = graph + + if not self.lm: + graph_with_and = pynutil.add_weight(graph, 0.00001) + not_quote = pynini.closure(NEMO_NOT_QUOTE) + no_thousand_million = pynini.difference( + not_quote, not_quote + pynini.union("thousand", "million") + not_quote + ).optimize() + integer = ( + not_quote + pynutil.add_weight(pynini.cross("hundred ", "hundred and ") + no_thousand_million, -0.0001) + ).optimize() + + no_hundred = pynini.difference(NEMO_SIGMA, not_quote + pynini.accep("hundred") + not_quote).optimize() + integer |= ( + not_quote + pynutil.add_weight(pynini.cross("thousand ", "thousand and ") + no_hundred, -0.0001) + ).optimize() + + optional_hundred = pynini.compose((NEMO_DIGIT - "0") ** 3, graph).optimize() + optional_hundred = pynini.compose(optional_hundred, NEMO_SIGMA + pynini.cross(" hundred", "") + NEMO_SIGMA) + graph_with_and |= pynini.compose(graph, integer).optimize() + graph_with_and |= optional_hundred + return graph_with_and diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..e01af1c70a1f30e39b44ded09f2404657c8fa516 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/date.py @@ -0,0 +1,386 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_LOWER, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + +graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/number/teen.tsv"))).optimize() +graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() +ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/number/ty.tsv"))).optimize() +year_suffix = load_labels(get_abs_path("data/date/year_suffix.tsv")) +year_suffix.extend(augment_labels_with_punct_at_end(year_suffix)) +year_suffix = pynini.string_map(year_suffix).optimize() + + +def get_ties_graph(deterministic: bool = True): + """ + Returns two digit transducer, e.g. + 03 -> o three + 12 -> thirteen + 20 -> twenty + """ + graph = graph_teen | ties_graph + pynutil.delete("0") | ties_graph + insert_space + graph_digit + + if deterministic: + graph = graph | pynini.cross("0", "oh") + insert_space + graph_digit + else: + graph = graph | (pynini.cross("0", "oh") | pynini.cross("0", "zero")) + insert_space + graph_digit + + return graph.optimize() + + +def get_four_digit_year_graph(deterministic: bool = True): + """ + Returns a four digit transducer which is combination of ties/teen or digits + (using hundred instead of thousand format), e.g. + 1219 -> twelve nineteen + 3900 -> thirty nine hundred + """ + graph_ties = get_ties_graph(deterministic) + + graph_with_s = ( + (graph_ties + insert_space + graph_ties) + | (graph_teen + insert_space + (ties_graph | pynini.cross("1", "ten"))) + ) + pynutil.delete("0s") + + graph_with_s |= (graph_teen | graph_ties) + insert_space + pynini.cross("00", "hundred") + pynutil.delete("s") + graph_with_s = graph_with_s @ pynini.cdrewrite( + pynini.cross("y", "ies") | pynutil.insert("s"), "", "[EOS]", NEMO_SIGMA + ) + + graph = graph_ties + insert_space + graph_ties + graph |= (graph_teen | graph_ties) + insert_space + pynini.cross("00", "hundred") + + thousand_graph = ( + graph_digit + + insert_space + + pynini.cross("00", "thousand") + + (pynutil.delete("0") | insert_space + graph_digit) + ) + thousand_graph |= ( + graph_digit + + insert_space + + pynini.cross("000", "thousand") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynini.accep("s") + ) + + graph |= graph_with_s + if deterministic: + graph = plurals._priority_union(thousand_graph, graph, NEMO_SIGMA) + else: + graph |= thousand_graph + + return graph.optimize() + + +def _get_two_digit_year_with_s_graph(): + # to handle '70s -> seventies + graph = ( + pynini.closure(pynutil.delete("'"), 0, 1) + + pynini.compose( + ties_graph + pynutil.delete("0s"), pynini.cdrewrite(pynini.cross("y", "ies"), "", "[EOS]", NEMO_SIGMA) + ) + ).optimize() + return graph + + +def _get_year_graph(cardinal_graph, deterministic: bool = True): + """ + Transducer for year, only from 1000 - 2999 e.g. + 1290 -> twelve nineteen + 2000 - 2009 will be verbalized as two thousand. + + Transducer for 3 digit year, e.g. 123-> one twenty three + + Transducer for year with suffix + 123 A.D., 4200 B.C + """ + graph = get_four_digit_year_graph(deterministic) + graph = (pynini.union("1", "2") + (NEMO_DIGIT**3) + pynini.closure(pynini.cross(" s", "s") | "s", 0, 1)) @ graph + + graph |= _get_two_digit_year_with_s_graph() + + three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT**2) @ cardinal_graph + year_with_suffix = ( + (get_four_digit_year_graph(deterministic=True) | three_digit_year) + delete_space + insert_space + year_suffix + ) + graph |= year_with_suffix + return graph.optimize() + + +def _get_two_digit_year(cardinal_graph, single_digits_graph): + two_digit_year = NEMO_DIGIT ** (2) @ plurals._priority_union(cardinal_graph, single_digits_graph, NEMO_SIGMA) + return two_digit_year + + +def _get_financial_period_graph(): + # 1H23 -> first half of twenty three + # 3Q22 -> third quarter of twenty two + + h_ordinals = pynini.cross('1', 'first') | pynini.cross('2', 'second') + q_ordinals = h_ordinals | pynini.cross('3', 'third') | pynini.cross('4', 'fourth') + + h_graph = h_ordinals + pynini.cross('H', ' half') + q_graph = q_ordinals + pynini.cross('Q', ' quarter') + period_graph = h_graph | q_graph + + return period_graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + jan. 5, 2012 -> date { month: "january" day: "five" year: "twenty twelve" preserve_order: true } + jan. 5 -> date { month: "january" day: "five" preserve_order: true } + 5 january 2012 -> date { day: "five" month: "january" year: "twenty twelve" preserve_order: true } + 2012-01-05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012.01.05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012/01/05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012 -> date { year: "twenty twelve" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool, lm: bool = False): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + # january + month_graph = pynini.string_file(get_abs_path("data/date/month_name.tsv")).optimize() + # January, JANUARY + month_graph |= pynini.compose(TO_LOWER + pynini.closure(NEMO_CHAR), month_graph) | pynini.compose( + TO_LOWER ** (2, ...), month_graph + ) + + # jan + month_abbr_graph = pynini.string_file(get_abs_path("data/date/month_abbr.tsv")).optimize() + # jan, Jan, JAN + month_abbr_graph = ( + month_abbr_graph + | pynini.compose(TO_LOWER + pynini.closure(NEMO_LOWER, 1), month_abbr_graph).optimize() + | pynini.compose(TO_LOWER ** (2, ...), month_abbr_graph).optimize() + ) + pynini.closure(pynutil.delete("."), 0, 1) + month_graph |= month_abbr_graph.optimize() + + month_numbers_labels = pynini.string_file(get_abs_path("data/date/month_number.tsv")).optimize() + cardinal_graph = cardinal.graph_hundred_component_at_least_one_none_zero_digit + + year_graph = _get_year_graph(cardinal_graph=cardinal_graph, deterministic=deterministic) + + # three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT ** 2) @ cardinal_graph + # year_graph |= three_digit_year + + month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + month_numbers_graph = pynutil.insert("month: \"") + month_numbers_labels + pynutil.insert("\"") + + endings = ["rd", "th", "st", "nd"] + endings += [x.upper() for x in endings] + endings = pynini.union(*endings) + + day_graph = ( + pynutil.insert("day: \"") + + pynini.closure(pynutil.delete("the "), 0, 1) + + ( + ((pynini.union("1", "2") + NEMO_DIGIT) | NEMO_DIGIT | (pynini.accep("3") + pynini.union("0", "1"))) + + pynini.closure(pynutil.delete(endings), 0, 1) + ) + @ cardinal_graph + + pynutil.insert("\"") + ) + + two_digit_year = _get_two_digit_year( + cardinal_graph=cardinal_graph, single_digits_graph=cardinal.single_digits_graph + ) + two_digit_year = pynutil.insert("year: \"") + two_digit_year + pynutil.insert("\"") + + # if lm: + # two_digit_year = pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (3), two_digit_year) + # year_graph = pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (2), year_graph) + # year_graph |= pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (4, ...), year_graph) + + graph_year = pynutil.insert(" year: \"") + pynutil.delete(" ") + year_graph + pynutil.insert("\"") + graph_year |= ( + pynutil.insert(" year: \"") + + pynini.accep(",") + + pynini.closure(pynini.accep(" "), 0, 1) + + year_graph + + pynutil.insert("\"") + ) + optional_graph_year = pynini.closure(graph_year, 0, 1) + + year_graph = pynutil.insert("year: \"") + year_graph + pynutil.insert("\"") + + graph_mdy = month_graph + ( + (delete_extra_space + day_graph) + | (pynini.accep(" ") + day_graph) + | graph_year + | (delete_extra_space + day_graph + graph_year) + ) + + graph_mdy |= ( + month_graph + + pynini.cross("-", " ") + + day_graph + + pynini.closure(((pynini.cross("-", " ") + NEMO_SIGMA) @ graph_year), 0, 1) + ) + + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_mdy |= ( + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + + delete_sep + + insert_space + + (year_graph | two_digit_year) + ) + + graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year + day_ex_month = (NEMO_DIGIT**2 - pynini.project(month_numbers_graph, "input")) @ day_graph + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_dmy |= ( + day_ex_month + + delete_sep + + insert_space + + month_numbers_graph + + delete_sep + + insert_space + + (year_graph | two_digit_year) + ) + + graph_ymd = pynini.accep("") + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_ymd |= ( + (year_graph | two_digit_year) + + delete_sep + + insert_space + + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + ) + + final_graph = graph_mdy | graph_dmy + + if not deterministic or lm: + final_graph += pynini.closure(pynutil.insert(" preserve_order: true"), 0, 1) + m_sep_d = ( + month_numbers_graph + + pynutil.delete(pynini.union("-", "/")) + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + ) + final_graph |= m_sep_d + else: + final_graph += pynutil.insert(" preserve_order: true") + + period_fy = pynutil.insert("text: \"") + _get_financial_period_graph() + pynutil.insert("\"") + graph_fy = period_fy + insert_space + two_digit_year + + final_graph |= graph_ymd | year_graph | graph_fy + + if not deterministic or lm: + ymd_to_mdy_graph = None + ymd_to_dmy_graph = None + mdy_to_dmy_graph = None + md_to_dm_graph = None + + for month in [x[0] for x in load_labels(get_abs_path("data/date/month_name.tsv"))]: + for day in [x[0] for x in load_labels(get_abs_path("data/date/day.tsv"))]: + ymd_to_mdy_curr = ( + pynutil.insert("month: \"" + month + "\" day: \"" + day + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + + pynutil.delete(" month: \"" + month + "\" day: \"" + day + "\"") + ) + + # YY-MM-DD -> MM-DD-YY + ymd_to_mdy_curr = pynini.compose(graph_ymd, ymd_to_mdy_curr) + ymd_to_mdy_graph = ( + ymd_to_mdy_curr + if ymd_to_mdy_graph is None + else pynini.union(ymd_to_mdy_curr, ymd_to_mdy_graph) + ) + + ymd_to_dmy_curr = ( + pynutil.insert("day: \"" + day + "\" month: \"" + month + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + + pynutil.delete(" month: \"" + month + "\" day: \"" + day + "\"") + ) + + # YY-MM-DD -> MM-DD-YY + ymd_to_dmy_curr = pynini.compose(graph_ymd, ymd_to_dmy_curr).optimize() + ymd_to_dmy_graph = ( + ymd_to_dmy_curr + if ymd_to_dmy_graph is None + else pynini.union(ymd_to_dmy_curr, ymd_to_dmy_graph) + ) + + mdy_to_dmy_curr = ( + pynutil.insert("day: \"" + day + "\" month: \"" + month + "\" ") + + pynutil.delete("month: \"" + month + "\" day: \"" + day + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + ).optimize() + # MM-DD-YY -> verbalize as MM-DD-YY (February fourth 1991) or DD-MM-YY (the fourth of February 1991) + mdy_to_dmy_curr = pynini.compose(graph_mdy, mdy_to_dmy_curr).optimize() + mdy_to_dmy_graph = ( + mdy_to_dmy_curr + if mdy_to_dmy_graph is None + else pynini.union(mdy_to_dmy_curr, mdy_to_dmy_graph).optimize() + ).optimize() + + md_to_dm_curr = pynutil.insert("day: \"" + day + "\" month: \"" + month + "\"") + pynutil.delete( + "month: \"" + month + "\" day: \"" + day + "\"" + ) + md_to_dm_curr = pynini.compose(m_sep_d, md_to_dm_curr).optimize() + + md_to_dm_graph = ( + md_to_dm_curr + if md_to_dm_graph is None + else pynini.union(md_to_dm_curr, md_to_dm_graph).optimize() + ).optimize() + + final_graph |= mdy_to_dmy_graph | md_to_dm_graph | ymd_to_mdy_graph | ymd_to_dmy_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..9c72566d2a2e39560e759926d6f142366678ff5b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py @@ -0,0 +1,130 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, TO_UPPER, GraphFst, get_abs_path + +delete_space = pynutil.delete(" ") +quantities = pynini.string_file(get_abs_path("data/number/thousand.tsv")) +quantities_abbr = pynini.string_file(get_abs_path("data/number/quantity_abbr.tsv")) +quantities_abbr |= TO_UPPER @ quantities_abbr + + +def get_quantity( + decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike', include_abbr: bool +) -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 1 million -> integer_part: "one" quantity: "million" + e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + quantity_wo_thousand = pynini.project(quantities, "input") - pynini.union("k", "K", "thousand") + if include_abbr: + quantity_wo_thousand |= pynini.project(quantities_abbr, "input") - pynini.union("k", "K", "thousand") + res = ( + pynutil.insert("integer_part: \"") + + cardinal_up_to_hundred + + pynutil.insert("\"") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert(" quantity: \"") + + (quantity_wo_thousand @ (quantities | quantities_abbr)) + + pynutil.insert("\"") + ) + if include_abbr: + quantity = quantities | quantities_abbr + else: + quantity = quantities + res |= ( + decimal + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert("quantity: \"") + + quantity + + pynutil.insert("\"") + ) + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" } + 1 billion -> decimal { integer_part: "one" quantity: "billion" } + + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph_with_and + cardinal_graph_hundred_component_at_least_one_none_zero_digit = ( + cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + + self.graph = cardinal.single_digits_graph.optimize() + + if not deterministic: + self.graph = self.graph | cardinal_graph + + point = pynutil.delete(".") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(self.graph_integer + pynutil.insert(" "), 0, 1) + + point + + pynutil.insert(" ") + + self.graph_fractional + ) + + quantity_w_abbr = get_quantity( + final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=True + ) + quantity_wo_abbr = get_quantity( + final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=False + ) + self.final_graph_wo_negative_w_abbr = final_graph_wo_sign | quantity_w_abbr + self.final_graph_wo_negative = final_graph_wo_sign | quantity_wo_abbr + + # reduce options for non_deterministic and allow either "oh" or "zero", but not combination + if not deterministic: + no_oh_zero = pynini.difference( + NEMO_SIGMA, + (NEMO_SIGMA + "oh" + NEMO_SIGMA + "zero" + NEMO_SIGMA) + | (NEMO_SIGMA + "zero" + NEMO_SIGMA + "oh" + NEMO_SIGMA), + ).optimize() + no_zero_oh = pynini.difference( + NEMO_SIGMA, NEMO_SIGMA + pynini.accep("zero") + NEMO_SIGMA + pynini.accep("oh") + NEMO_SIGMA + ).optimize() + + self.final_graph_wo_negative |= pynini.compose( + self.final_graph_wo_negative, + pynini.cdrewrite( + pynini.cross("integer_part: \"zero\"", "integer_part: \"oh\""), NEMO_SIGMA, NEMO_SIGMA, NEMO_SIGMA + ), + ) + self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_oh_zero).optimize() + self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_zero_oh).optimize() + + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..55f5d20e19fd7474bfdbe10a37ebc9765036908a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -0,0 +1,179 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_UPPER, + TO_UPPER, + GraphFst, + get_abs_path, + insert_space, +) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. cdf1@abc.edu -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + if deterministic: + numbers = NEMO_DIGIT + else: + numbers = pynutil.insert(" ") + cardinal.long_numbers + pynutil.insert(" ") + + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + + accepted_symbols = pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "input") + + accepted_common_domains = pynini.project( + pynini.string_file(get_abs_path("data/electronic/domain.tsv")), "input" + ) + + dict_words = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/words.tsv")), + MIN_NEG_WEIGHT, + ) + + dict_words_without_delimiter = dict_words + pynini.closure( + pynutil.add_weight(pynutil.insert(" ") + dict_words, MIN_NEG_WEIGHT), + 1, + ) + dict_words_graph = dict_words_without_delimiter | dict_words + + all_accepted_symbols_start = ( + dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + ).optimize() + + all_accepted_symbols_end = ( + dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + ).optimize() + + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + username = (NEMO_ALPHA | dict_words_graph) + pynini.closure( + NEMO_ALPHA | numbers | accepted_symbols | dict_words_graph + ) + + username = pynutil.insert('username: "') + username + pynutil.insert('"') + pynini.cross("@", " ") + + domain_graph = all_accepted_symbols_start + pynini.closure( + all_accepted_symbols_end | pynutil.add_weight(accepted_common_domains, MIN_NEG_WEIGHT) + ) + + protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "colon")) + pynutil.insert(" ")) + + protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( + pynini.accep("://") @ protocol_symbols + ) + + protocol_file_start = pynini.accep("file") + insert_space + (pynini.accep(":///") @ protocol_symbols) + + protocol_end = pynutil.add_weight(pynini.cross("www", "WWW ") + pynini.accep(".") @ protocol_symbols, -1000) + + protocol = protocol_file_start | protocol_start | protocol_end | (protocol_start + protocol_end) + + domain_graph_with_class_tags = ( + pynutil.insert('domain: "') + + pynini.compose( + NEMO_ALPHA + pynini.closure(NEMO_NOT_SPACE) + (NEMO_ALPHA | NEMO_DIGIT | pynini.accep("/")), + domain_graph, + ).optimize() + + pynutil.insert('"') + ) + + protocol = pynutil.insert('protocol: "') + pynutil.add_weight(protocol, MIN_NEG_WEIGHT) + pynutil.insert('"') + + # email + graph = pynini.compose( + NEMO_SIGMA + pynini.accep("@") + NEMO_SIGMA + pynini.accep(".") + NEMO_SIGMA, + username + domain_graph_with_class_tags, + ) + + full_stop_accep = pynini.accep(".") + dollar_accep = pynini.accep("$") # Include for the correct transduction of the money graph + excluded_symbols = full_stop_accep | dollar_accep | pynini.accep(",") + filtered_symbols = pynini.difference(accepted_symbols, excluded_symbols) + accepted_characters = NEMO_ALPHA | NEMO_DIGIT | filtered_symbols + domain_component = full_stop_accep + pynini.closure(accepted_characters, 2) + graph_domain = pynutil.add_weight( + pynutil.insert('domain: "') + + (pynini.closure(accepted_characters, 1) + pynini.closure(domain_component, 1)) + + pynutil.insert('"'), + 0.1, + ).optimize() + + graph |= pynutil.add_weight(graph_domain, MIN_NEG_WEIGHT) + + # www.abc.com/sdafsdf, or https://www.abc.com/asdfad or www.abc.abc/asdfad + graph |= protocol + pynutil.insert(" ") + domain_graph_with_class_tags + + # recursively handles the "/" in strings like: + # update/upgrade -> update slash upgrade + # update/upgrade/downgrade -> update slash upgrade slash downgrade + + slash_string = ( + pynini.accep(" ").ques + pynini.accep("/") + pynini.accep(" ").ques + pynini.closure(NEMO_ALPHA, 1) + ) + + graph |= ( + pynutil.insert('domain: "') + + (pynini.closure(NEMO_ALPHA, 1) - "and") + + pynini.closure(slash_string, 1) + + pynutil.insert('"') + ).optimize() + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + + final_graph = self.add_tokens(graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..6c0b34053202ad51e75b8945ebbf5d2c2f827eae --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py @@ -0,0 +1,56 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } + "23 4/5th" -> + tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + numerator = ( + pynutil.insert("numerator: \"") + cardinal_graph + (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")) + ) + + endings = ["rd", "th", "st", "nd"] + endings += [x.upper() for x in endings] + optional_end = pynini.closure(pynini.cross(pynini.union(*endings), ""), 0, 1) + + denominator = pynutil.insert("denominator: \"") + cardinal_graph + optional_end + pynutil.insert("\"") + + graph = pynini.closure(integer + pynini.accep(" "), 0, 1) + (numerator + denominator) + graph |= pynini.closure(integer + (pynini.accep(" ") | pynutil.insert(" ")), 0, 1) + pynini.compose( + pynini.string_file(get_abs_path("data/number/fraction.tsv")), (numerator + denominator) + ) + + self.graph = graph + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..254688194558fdfce7e677bdb6feffa507b1a5f9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py @@ -0,0 +1,332 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_UPPER, + SINGULAR_TO_PLURAL, + TO_LOWER, + GraphFst, + convert_space, + delete_space, + delete_zero_or_one_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst as OrdinalTagger +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import get_formats +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as OrdinalVerbalizer + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, suppletive aware, e.g. + -12kg -> measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } + 1kg -> measure { cardinal { integer: "one" } units: "kilogram" } + .5kg -> measure { decimal { fractional_part: "five" } units: "kilograms" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph_with_and | self.get_range(cardinal.graph_with_and) + + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) + if not deterministic: + graph_unit |= pynini.string_file(get_abs_path("data/measure/unit_alternatives.tsv")) + + graph_unit |= pynini.compose( + pynini.closure(TO_LOWER, 1) + (NEMO_ALPHA | TO_LOWER) + pynini.closure(NEMO_ALPHA | TO_LOWER), + graph_unit, + ).optimize() + + graph_unit_plural = convert_space(graph_unit @ SINGULAR_TO_PLURAL) + graph_unit = convert_space(graph_unit) + + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + graph_unit2 = ( + pynini.cross("/", "per") + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit + ) + + optional_graph_unit2 = pynini.closure( + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, + 0, + 1, + ) + + unit_plural = ( + pynutil.insert('units: "') + (graph_unit_plural + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') + ) + + unit_singular = ( + pynutil.insert('units: "') + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + unit_plural + ) + + # support radio FM/AM + subgraph_decimal |= ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + pynutil.insert('units: "') + + pynini.union("AM", "FM") + + pynutil.insert('"') + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + ((NEMO_SIGMA - "1") @ cardinal_graph) + + delete_space + + pynutil.insert('"') + + pynutil.insert(" } ") + + unit_plural + ) + + subgraph_cardinal |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + pynini.cross("1", "one") + + delete_space + + pynutil.insert('"') + + pynutil.insert(" } ") + + unit_singular + ) + + unit_graph = ( + pynutil.insert('cardinal { integer: "-" } units: "') + + ((pynini.cross("/", "per") + delete_zero_or_one_space) | (pynini.accep("per") + pynutil.delete(" "))) + + pynutil.insert(NEMO_NON_BREAKING_SPACE) + + graph_unit + + pynutil.insert('" preserve_order: true') + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynini.cross("-", "") + + pynutil.insert(' } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert('"') + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(' } units: "') + + (pynini.cross(pynini.union("x", "X"), "x") | pynini.cross(pynini.union("x", "X"), " times")) + + pynutil.insert('"') + ) + + alpha_dash_decimal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynini.accep("-") + + pynutil.insert('"') + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + subgraph_fraction = ( + pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit_plural + ) + + address = self.get_address_graph(cardinal) + address = ( + pynutil.insert('units: "address" cardinal { integer: "') + + address + + pynutil.insert('" } preserve_order: true') + ) + + math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + + math = ( + (cardinal_graph | NEMO_ALPHA) + + delimiter + + math_operations + + (delimiter | NEMO_ALPHA) + + cardinal_graph + + delimiter + + pynini.cross("=", "equals") + + delimiter + + (cardinal_graph | NEMO_ALPHA) + ) + + math |= ( + (cardinal_graph | NEMO_ALPHA) + + delimiter + + pynini.cross("=", "equals") + + delimiter + + (cardinal_graph | NEMO_ALPHA) + + delimiter + + math_operations + + delimiter + + cardinal_graph + ) + + math = ( + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') + ) + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | unit_graph + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | subgraph_fraction + | address + | math + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def get_range(self, cardinal: GraphFst): + """ + Returns range forms for measure tagger, e.g. 2-3, 2x3, 2*2 + + Args: + cardinal: cardinal GraphFst + """ + range_graph = cardinal + pynini.cross(pynini.union("-", " - "), " to ") + cardinal + + for x in [" x ", "x"]: + range_graph |= cardinal + pynini.cross(x, " by ") + cardinal + if not self.deterministic: + range_graph |= cardinal + pynini.cross(x, " times ") + pynini.closure(cardinal, 0, 1) + + for x in ["*", " * "]: + range_graph |= cardinal + pynini.cross(x, " times ") + cardinal + return range_graph.optimize() + + def get_address_graph(self, cardinal): + """ + Finite state transducer for classifying serial. + The serial is a combination of digits, letters and dashes, e.g.: + 2788 San Tomas Expy, Santa Clara, CA 95051 -> + units: "address" cardinal + { integer: "two seven eight eight San Tomas Expressway Santa Clara California nine five zero five one" } + preserve_order: true + """ + ordinal_verbalizer = OrdinalVerbalizer().graph + ordinal_tagger = OrdinalTagger(cardinal=cardinal).graph + ordinal_num = pynini.compose( + pynutil.insert('integer: "') + ordinal_tagger + pynutil.insert('"'), + ordinal_verbalizer, + ) + + address_num = NEMO_DIGIT ** (1, 2) @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + address_num += insert_space + NEMO_DIGIT**2 @ ( + pynini.closure(pynini.cross("0", "zero "), 0, 1) + + cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + # to handle the rest of the numbers + address_num = pynini.compose(NEMO_DIGIT ** (3, 4), address_num) + address_num = plurals._priority_union(address_num, cardinal.graph, NEMO_SIGMA) + + direction = ( + pynini.cross("E", "East") + | pynini.cross("S", "South") + | pynini.cross("W", "West") + | pynini.cross("N", "North") + ) + pynini.closure(pynutil.delete("."), 0, 1) + + direction = pynini.closure(pynini.accep(NEMO_SPACE) + direction, 0, 1) + address_words = get_formats(get_abs_path("data/address/address_word.tsv")) + address_words = ( + pynini.accep(NEMO_SPACE) + + (pynini.closure(ordinal_num, 0, 1) | NEMO_UPPER + pynini.closure(NEMO_ALPHA, 1)) + + NEMO_SPACE + + pynini.closure(NEMO_UPPER + pynini.closure(NEMO_ALPHA) + NEMO_SPACE) + + address_words + ) + + city = pynini.closure(NEMO_ALPHA | pynini.accep(NEMO_SPACE), 1) + city = pynini.closure(pynini.accep(",") + pynini.accep(NEMO_SPACE) + city, 0, 1) + + states = load_labels(get_abs_path("data/address/state.tsv")) + + additional_options = [] + for x, y in states: + additional_options.append((x, f"{y[0]}.{y[1:]}")) + states.extend(additional_options) + state_graph = pynini.string_map(states) + state = pynini.invert(state_graph) + state = pynini.closure(pynini.accep(",") + pynini.accep(NEMO_SPACE) + state, 0, 1) + + zip_code = pynini.compose(NEMO_DIGIT**5, cardinal.single_digits_graph) + zip_code = pynini.closure( + pynini.closure(pynini.accep(","), 0, 1) + pynini.accep(NEMO_SPACE) + zip_code, + 0, + 1, + ) + + address = address_num + direction + address_words + pynini.closure(city + state + zip_code, 0, 1) + + # Handles sentences ending in terminal period abbreviations, when followed by another sentence + # (e.g. Nancy lived on 1428 Elm St. It was a strange place-> Nancy lived on fourteen twenty eight Elm Street. It was a strange place.) + + hyphen = pynini.accep("-") + alpha_chars = NEMO_ALPHA | hyphen + + address_eos = ( + address_num + + direction + + address_words + + pynini.accep(".") + + pynini.closure(NEMO_SPACE, 1, 2) + + NEMO_UPPER + + pynini.closure(alpha_chars) + ) + + address |= pynutil.add_weight(address_eos, -0.001) + + address |= address_num + direction + address_words + pynini.closure(pynini.cross(".", ""), 0, 1) + + return address diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..9e1b917f43e0dc6ecb7dcf5532fdaa75e6d626ad --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/money.py @@ -0,0 +1,199 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) +min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, suppletive aware, e.g. + $12.05 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } + $12.0500 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } + $1 -> money { currency_maj: "dollar" integer_part: "one" } + $1.00 -> money { currency_maj: "dollar" integer_part: "one" } + $0.05 -> money { fractional_part: "five" currency_min: "cents" preserve_order: true } + $1 million -> money { currency_maj: "dollars" integer_part: "one" quantity: "million" } + $1.2 million -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two" quantity: "million" } + $1.2320 -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two three two" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph_with_and + graph_decimal_final = decimal.final_graph_wo_negative_w_abbr + + maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + maj_unit_plural = convert_space(maj_singular @ SINGULAR_TO_PLURAL) + maj_unit_singular = convert_space(maj_singular) + + graph_maj_singular = pynutil.insert('currency_maj: "') + maj_unit_singular + pynutil.insert('"') + graph_maj_plural = pynutil.insert('currency_maj: "') + maj_unit_plural + pynutil.insert('"') + + optional_delete_fractional_zeros = pynini.closure( + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), 0, 1 + ) + + graph_integer_one = pynutil.insert('integer_part: "') + pynini.cross("1", "one") + pynutil.insert('"') + # only for decimals where third decimal after comma is non-zero or with quantity + decimal_delete_last_zeros = ( + pynini.closure(NEMO_DIGIT | pynutil.delete(",")) + + pynini.accep(".") + + pynini.closure(NEMO_DIGIT, 2) + + (NEMO_DIGIT - "0") + + pynini.closure(pynutil.delete("0")) + ) + decimal_with_quantity = NEMO_SIGMA + NEMO_ALPHA + + graph_decimal = ( + graph_maj_plural + insert_space + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + ) + + graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"') + + graph_integer_only = graph_maj_singular + insert_space + graph_integer_one + graph_integer_only |= graph_maj_plural + insert_space + graph_integer + + final_graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"') + graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"') + # format ** dollars ** cent + decimal_graph_with_minor = None + integer_graph_reordered = None + decimal_default_reordered = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + integer_plus_maj = graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural + integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular + + integer_plus_maj_with_comma = pynini.compose( + NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), + integer_plus_maj, + ) + integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj) + integer_plus_maj |= integer_plus_maj_with_comma + + graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "one") + graph_fractional_one = pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"') + graph_fractional = ( + two_digits_fractional_part + @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") + @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"') + + fractional_plus_min = graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural + fractional_plus_min |= ( + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular + ) + + decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(".", " ") + fractional_plus_min + + if not deterministic: + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(".", " ") + + pynutil.insert('fractional_part: "') + + two_digits_fractional_part @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + + pynutil.insert('"'), + weight=0.0001, + ) + default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + decimal_graph_with_minor_curr |= ( + pynini.closure(pynutil.delete("0"), 0, 1) + pynutil.delete(".") + fractional_plus_min + ) + decimal_graph_with_minor_curr = ( + pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr).optimize() + ) + + if not deterministic: + integer_graph_reordered_curr = ( + pynutil.delete(curr_symbol) + integer_plus_maj + preserve_order + ).optimize() + + integer_graph_reordered = ( + integer_graph_reordered_curr + if integer_graph_reordered is None + else pynini.union(integer_graph_reordered, integer_graph_reordered_curr).optimize() + ) + decimal_default_reordered_curr = ( + pynutil.delete(curr_symbol) + + default_fraction_graph + + insert_space + + pynutil.insert(curr_symbol) @ graph_maj_plural + ) + + decimal_default_reordered = ( + decimal_default_reordered_curr + if decimal_default_reordered is None + else pynini.union(decimal_default_reordered, decimal_default_reordered_curr) + ).optimize() + + # weight for SH + final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001) + + # utilizes morphosyntactic features to append "per units" + graph_per_units = pynutil.insert(' morphosyntactic_features: "') + per_units + pynutil.insert('"') + + if not deterministic: + final_graph |= integer_graph_reordered | decimal_default_reordered + # to handle "$2.00" cases + final_graph |= pynini.compose( + NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), + integer_graph_reordered, + ) + + final_graph += graph_per_units.ques + final_graph = self.add_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..b17ed91471176cc075162784be4cea8ebb084e36 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py @@ -0,0 +1,62 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 13th -> ordinal { integer: "thirteen" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + cardinal_format = pynini.closure(NEMO_DIGIT | pynini.accep(",")) + st_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("1") + + pynutil.delete(pynini.union("st", "ST", "ˢᵗ")) + ) + nd_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("2") + + pynutil.delete(pynini.union("nd", "ND", "ⁿᵈ")) + ) + rd_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("3") + + pynutil.delete(pynini.union("rd", "RD", "ʳᵈ")) + ) + th_format = pynini.closure( + (NEMO_DIGIT - "1" - "2" - "3") + | (cardinal_format + "1" + NEMO_DIGIT) + | (cardinal_format + (NEMO_DIGIT - "1") + (NEMO_DIGIT - "1" - "2" - "3")), + 1, + ) + pynutil.delete(pynini.union("th", "TH", "ᵗʰ")) + self.graph = (st_format | nd_format | rd_format | th_format) @ cardinal_graph + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..0f7a4b74337f2354fa5ac7b724af17cf5981d643 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py @@ -0,0 +1,66 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + [r"\[", r"\]"] + + whitelist_symbols = load_labels(get_abs_path("data/whitelist/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/range.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/range.py new file mode 100644 index 0000000000000000000000000000000000000000..f84f4acd851c898940c94caf633c5e8da839834e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/range.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space + + +class RangeFst(GraphFst): + """ + This class is a composite class of two other class instances + + Args: + time: composed tagger and verbalizer + date: composed tagger and verbalizer + cardinal: tagger + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + lm: whether to use for hybrid LM + """ + + def __init__( + self, + time: GraphFst, + date: GraphFst, + cardinal: GraphFst, + deterministic: bool = True, + lm: bool = False, + ): + super().__init__(name="range", kind="classify", deterministic=deterministic) + + delete_space = pynini.closure(pynutil.delete(" "), 0, 1) + + approx = pynini.cross("~", "approximately") + + # TIME + time_graph = time + delete_space + pynini.cross("-", " to ") + delete_space + time + self.graph = time_graph | (approx + time) + + cardinal = cardinal.graph_with_and + # YEAR + date_year_four_digit = (NEMO_DIGIT**4 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + date_year_two_digit = (NEMO_DIGIT**2 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + year_to_year_graph = ( + date_year_four_digit + + delete_space + + pynini.cross("-", " to ") + + delete_space + + (date_year_four_digit | date_year_two_digit | (NEMO_DIGIT**2 @ cardinal)) + ) + mid_year_graph = pynini.accep("mid") + pynini.cross("-", " ") + (date_year_four_digit | date_year_two_digit) + + self.graph |= year_to_year_graph + self.graph |= mid_year_graph + + # ADDITION + range_graph = cardinal + pynini.closure(pynini.cross("+", " plus ") + cardinal, 1) + range_graph |= cardinal + pynini.closure(pynini.cross(" + ", " plus ") + cardinal, 1) + range_graph |= approx + cardinal + range_graph |= cardinal + (pynini.cross("...", " ... ") | pynini.accep(" ... ")) + cardinal + + if not deterministic or lm: + # cardinal ---- + cardinal_to_cardinal_graph = ( + cardinal + delete_space + pynini.cross("-", pynini.union(" to ", " minus ")) + delete_space + cardinal + ) + + range_graph |= cardinal_to_cardinal_graph | ( + cardinal + delete_space + pynini.cross(":", " to ") + delete_space + cardinal + ) + + # MULTIPLY + for x in [" x ", "x"]: + range_graph |= cardinal + pynini.cross(x, pynini.union(" by ", " times ")) + cardinal + + # 40x -> "40 times" ("40 x" cases is covered in serial) + for x in [" x", "x"]: + range_graph |= cardinal + pynini.cross(x, " times") + + # 5x to 7x-> five to seven x/times + range_graph |= ( + cardinal + + pynutil.delete(x) + + pynini.union(" to ", "-", " - ") + + cardinal + + pynini.cross(x, pynini.union(" x", " times")) + ) + + for x in ["*", " * "]: + range_graph |= cardinal + pynini.closure(pynini.cross(x, " times ") + cardinal, 1) + + # supports "No. 12" -> "Number 12" + range_graph |= ( + (pynini.cross(pynini.union("NO", "No"), "Number") | pynini.cross("no", "number")) + + pynini.closure(pynini.union(". ", " "), 0, 1) + + cardinal + ) + + for x in ["/", " / "]: + range_graph |= cardinal + pynini.closure(pynini.cross(x, " divided by ") + cardinal, 1) + + # 10% to 20% -> ten to twenty percent + range_graph |= ( + cardinal + + pynini.closure(pynini.cross("%", " percent") | pynutil.delete("%"), 0, 1) + + pynini.union(" to ", "-", " - ") + + cardinal + + pynini.cross("%", " percent") + ) + + self.graph |= range_graph + + self.graph = self.graph.optimize() + graph = pynutil.insert("name: \"") + convert_space(self.graph).optimize() + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py new file mode 100644 index 0000000000000000000000000000000000000000..23b9886f197ff65eed267d41f308c3f6e7df169b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying roman numbers: + e.g. "IV" -> tokens { roman { integer: "four" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + roman_dict = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) + default_graph = pynini.string_map(roman_dict).optimize() + default_graph = pynutil.insert("integer: \"") + default_graph + pynutil.insert("\"") + ordinal_limit = 19 + + if deterministic: + # exclude "I" + start_idx = 1 + else: + start_idx = 0 + + graph_teens = pynini.string_map([x[0] for x in roman_dict[start_idx:ordinal_limit]]).optimize() + + # roman numerals up to ordinal_limit with a preceding name are converted to ordinal form + names = get_names() + graph = ( + pynutil.insert("key_the_ordinal: \"") + + names + + pynutil.insert("\"") + + pynini.accep(" ") + + graph_teens @ default_graph + ).optimize() + + # single symbol roman numerals with preceding key words (multiple formats) are converted to cardinal form + key_words = [] + for k_word in load_labels(get_abs_path("data/roman/key_word.tsv")): + key_words.append(k_word) + key_words.append([k_word[0][0].upper() + k_word[0][1:]]) + key_words.append([k_word[0].upper()]) + + key_words = pynini.string_map(key_words).optimize() + graph |= ( + pynutil.insert("key_cardinal: \"") + key_words + pynutil.insert("\"") + pynini.accep(" ") + default_graph + ).optimize() + + if deterministic or lm: + # two digit roman numerals up to 49 + roman_to_cardinal = pynini.compose( + pynini.closure(NEMO_ALPHA, 2), + ( + pynutil.insert("default_cardinal: \"default\" ") + + (pynini.string_map([x[0] for x in roman_dict[:50]]).optimize()) @ default_graph + ), + ) + graph |= roman_to_cardinal + elif not lm: + # two or more digit roman numerals + roman_to_cardinal = pynini.compose( + pynini.difference(NEMO_SIGMA, "I"), + ( + pynutil.insert("default_cardinal: \"default\" integer: \"") + + pynini.string_map(roman_dict).optimize() + + pynutil.insert("\"") + ), + ).optimize() + graph |= roman_to_cardinal + + # convert three digit roman or up with suffix to ordinal + roman_to_ordinal = pynini.compose( + pynini.closure(NEMO_ALPHA, 3), + (pynutil.insert("default_ordinal: \"default\" ") + graph_teens @ default_graph + pynutil.delete("th")), + ) + + graph |= roman_to_ordinal + graph = self.add_tokens(graph.optimize()) + + self.fst = graph.optimize() + + +def get_names(): + """ + Returns the graph that matched common male and female names. + """ + male_labels = load_labels(get_abs_path("data/roman/male.tsv")) + female_labels = load_labels(get_abs_path("data/roman/female.tsv")) + male_labels.extend([[x[0].upper()] for x in male_labels]) + female_labels.extend([[x[0].upper()] for x in female_labels]) + names = pynini.string_map(male_labels).optimize() + names |= pynini.string_map(female_labels).optimize() + return names diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py new file mode 100644 index 0000000000000000000000000000000000000000..a37e79328fe2126d855ff619bb49fd70aa90336f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -0,0 +1,143 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class SerialFst(GraphFst): + """ + This class is a composite class of two other class instances + + Args: + time: composed tagger and verbalizer + date: composed tagger and verbalizer + cardinal: tagger + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + lm: whether to use for hybrid LM + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="integer", kind="classify", deterministic=deterministic) + + """ + Finite state transducer for classifying serial (handles only cases without delimiters, + values with delimiters are handled by default). + The serial is a combination of digits, letters and dashes, e.g.: + c325b -> tokens { cardinal { integer: "c three two five b" } } + """ + if deterministic: + num_graph = pynini.compose(NEMO_DIGIT ** (6, ...), cardinal.single_digits_graph).optimize() + num_graph |= pynini.compose(NEMO_DIGIT ** (1, 5), cardinal.graph).optimize() + # to handle numbers starting with zero + num_graph |= pynini.compose( + pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph + ).optimize() + else: + num_graph = cardinal.final_graph + + # TODO: "#" doesn't work from the file + symbols_graph = pynini.string_file(get_abs_path("data/whitelist/symbol.tsv")).optimize() | pynini.cross( + "#", "hash" + ) + num_graph |= symbols_graph + + if not self.deterministic and not lm: + num_graph |= cardinal.single_digits_graph + num_graph |= pynini.compose(num_graph, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA) + # also allow double digits to be pronounced as integer in serial number + num_graph |= pynutil.add_weight( + NEMO_DIGIT**2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=0.0001 + ) + + # add space between letter and digit/symbol + symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))] + symbols = pynini.union(*symbols) + digit_symbol = NEMO_DIGIT | symbols + + graph_with_space = pynini.compose( + pynini.cdrewrite(pynutil.insert(" "), NEMO_ALPHA | symbols, digit_symbol, NEMO_SIGMA), + pynini.cdrewrite(pynutil.insert(" "), digit_symbol, NEMO_ALPHA | symbols, NEMO_SIGMA), + ) + + # serial graph with delimiter + delimiter = pynini.accep("-") | pynini.accep("/") | pynini.accep(" ") + if not deterministic: + delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash ") + + alphas = pynini.closure(NEMO_ALPHA, 1) + letter_num = alphas + delimiter + num_graph + num_letter = pynini.closure(num_graph + delimiter, 1) + alphas + next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph)) + next_alpha_or_num |= pynini.closure( + delimiter + + num_graph + + plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize() + + alphas + ) + + serial_graph = letter_num + next_alpha_or_num + serial_graph |= num_letter + next_alpha_or_num + # numbers only with 2+ delimiters + serial_graph |= ( + num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph) + ) + # 2+ symbols + serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph) + + # exclude ordinal numbers from serial options + serial_graph = pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph + ).optimize() + + serial_graph = pynutil.add_weight(serial_graph, 0.0001) + serial_graph |= ( + pynini.closure(NEMO_NOT_SPACE, 1) + + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize() + ) + + # at least one serial graph with alpha numeric value and optional additional serial/num/alpha values + serial_graph = ( + pynini.closure((serial_graph | num_graph | alphas) + delimiter) + + serial_graph + + pynini.closure(delimiter + (serial_graph | num_graph | alphas)) + ) + + serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize() + serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() + + # this is not to verbolize "/" as "slash" in cases like "import/export" + serial_graph = pynini.compose( + pynini.difference( + NEMO_SIGMA, pynini.closure(NEMO_ALPHA, 1) + pynini.accep("/") + pynini.closure(NEMO_ALPHA, 1) + ), + serial_graph, + ) + + self.graph = serial_graph.optimize() + graph = pynutil.insert("name: \"") + convert_space(self.graph).optimize() + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..46632a0b28649a8a8d17793db2e4aac87c914fc7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py @@ -0,0 +1,134 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, + insert_space, + plurals, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension + country code optional: +*** + number part: ***-***-****, or (***) ***-**** + extension optional: 1-9999 + E.g + +1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + 1-800-GO-U-HAUL -> telephone { country_code: "one" number_part: "one, eight hundred GO U HAUL" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + add_separator = pynutil.insert(", ") # between components + zero = pynini.cross("0", "zero") + if not deterministic: + zero |= pynini.cross("0", pynini.union("o", "oh")) + digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() | zero + + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + country_code = ( + pynini.closure(telephone_prompts + delete_extra_space, 0, 1) + + pynini.closure(pynini.cross("+", "plus "), 0, 1) + + pynini.closure(digit + insert_space, 0, 2) + + digit + + pynutil.insert(",") + ) + country_code |= telephone_prompts + country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"") + country_code = country_code + pynini.closure(pynutil.delete("-"), 0, 1) + delete_space + insert_space + + area_part_default = pynini.closure(digit + insert_space, 2, 2) + digit + area_part = pynini.cross("800", "eight hundred") | pynini.compose( + pynini.difference(NEMO_SIGMA, "800"), area_part_default + ) + + area_part = ( + (area_part + (pynutil.delete("-") | pynutil.delete("."))) + | ( + pynutil.delete("(") + + area_part + + ((pynutil.delete(")") + pynini.closure(pynutil.delete(" "), 0, 1)) | pynutil.delete(")-")) + ) + ) + add_separator + + del_separator = pynini.closure(pynini.union("-", " ", "."), 0, 1) + number_length = ((NEMO_DIGIT + del_separator) | (NEMO_ALPHA + del_separator)) ** 7 + number_words = pynini.closure( + (NEMO_DIGIT @ digit) + (insert_space | (pynini.cross("-", ', '))) + | NEMO_ALPHA + | (NEMO_ALPHA + pynini.cross("-", ' ')) + ) + number_words |= pynini.closure( + (NEMO_DIGIT @ digit) + (insert_space | (pynini.cross(".", ', '))) + | NEMO_ALPHA + | (NEMO_ALPHA + pynini.cross(".", ' ')) + ) + number_words = pynini.compose(number_length, number_words) + number_part = area_part + number_words + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + extension = ( + pynutil.insert("extension: \"") + pynini.closure(digit + insert_space, 0, 3) + digit + pynutil.insert("\"") + ) + extension = pynini.closure(insert_space + extension, 0, 1) + + graph = plurals._priority_union(country_code + number_part, number_part, NEMO_SIGMA).optimize() + graph = plurals._priority_union(country_code + number_part + extension, graph, NEMO_SIGMA).optimize() + graph = plurals._priority_union(number_part + extension, graph, NEMO_SIGMA).optimize() + + # ip + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + digit_to_str_graph = digit + pynini.closure(pynutil.insert(" ") + digit, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " dot ") + digit_to_str_graph) ** 3 + graph |= ( + pynini.closure( + pynutil.insert("country_code: \"") + ip_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1 + ) + + pynutil.insert("number_part: \"") + + ip_graph.optimize() + + pynutil.insert("\"") + ) + # ssn + ssn_prompts = pynini.string_file(get_abs_path("data/telephone/ssn_prompt.tsv")) + three_digit_part = digit + (pynutil.insert(" ") + digit) ** 2 + two_digit_part = digit + pynutil.insert(" ") + digit + four_digit_part = digit + (pynutil.insert(" ") + digit) ** 3 + ssn_separator = pynini.cross("-", ", ") + ssn_graph = three_digit_part + ssn_separator + two_digit_part + ssn_separator + four_digit_part + + graph |= ( + pynini.closure( + pynutil.insert("country_code: \"") + ssn_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1 + ) + + pynutil.insert("number_part: \"") + + ssn_graph.optimize() + + pynutil.insert("\"") + ) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..9dc4849fc04f11cfff534aed00073c13aed9abb3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/time.py @@ -0,0 +1,133 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 12:30 a.m. est -> time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } + 2.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 02.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 2.00 a.m. -> time { hours: "two" suffix: "a m" } + 2 a.m. -> time { hours: "two" suffix: "a m" } + 02:00 -> time { hours: "two" } + 2:00 -> time { hours: "two" } + 10:00:05 a.m. -> time { hours: "ten" minutes: "zero" seconds: "five" suffix: "a m" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + suffix_labels = load_labels(get_abs_path("data/time/suffix.tsv")) + suffix_labels.extend(augment_labels_with_punct_at_end(suffix_labels)) + suffix_graph = pynini.string_map(suffix_labels) + + time_zone_graph = pynini.string_file(get_abs_path("data/time/zone.tsv")) + + # only used for < 1000 thousand -> 0 weight + cardinal = cardinal.graph + + labels_hour = [str(x) for x in range(0, 24)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | ( + pynini.closure(pynutil.delete("0"), 0, 1) + NEMO_DIGIT + ) + + graph_hour = delete_leading_zero_to_double_digit @ pynini.union(*labels_hour) @ cardinal + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + + final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynini.cross("0", "o") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynini.cross("0", "o") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_suffix = pynutil.insert("suffix: \"") + convert_space(suffix_graph) + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + final_time_zone_optional = pynini.closure( + delete_space + + insert_space + + pynutil.insert("zone: \"") + + convert_space(time_zone_graph) + + pynutil.insert("\""), + 0, + 1, + ) + + # 2:30 pm, 02:30, 2:00 + graph_hm = ( + final_graph_hour + + pynutil.delete(":") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + final_suffix_optional + + final_time_zone_optional + ) + + # 10:30:05 pm, + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"zero\"") | insert_space + final_graph_minute) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"zero\"") | insert_space + final_graph_second) + + final_suffix_optional + + final_time_zone_optional + ) + + # 2.xx pm/am + graph_hm2 = ( + final_graph_hour + + pynutil.delete(".") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + delete_space + + insert_space + + final_suffix + + final_time_zone_optional + ) + # 2 pm est + graph_h = final_graph_hour + delete_space + insert_space + final_suffix + final_time_zone_optional + final_graph = (graph_hm | graph_h | graph_hm2 | graph_hms).optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..e535e4b045fb15508a0d90b7394f9f088b1d775f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py @@ -0,0 +1,241 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.abbreviation import AbbreviationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDateFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTimeFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"en_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars.") + + start_time = time.time() + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + logger.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + + start_time = time.time() + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + logger.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") + + start_time = time.time() + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + logger.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") + + start_time = time.time() + fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) + fraction_graph = fraction.fst + logger.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") + + start_time = time.time() + measure = MeasureFst( + cardinal=cardinal, + decimal=decimal, + fraction=fraction, + deterministic=deterministic, + ) + measure_graph = measure.fst + logger.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + + start_time = time.time() + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") + + start_time = time.time() + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") + + start_time = time.time() + telephone_graph = TelephoneFst(deterministic=deterministic).fst + logger.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") + + start_time = time.time() + electonic_graph = ElectronicFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") + + start_time = time.time() + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + logger.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + + start_time = time.time() + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + logger.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + + start_time = time.time() + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + logger.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + + start_time = time.time() + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + logger.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + + start_time = time.time() + serial_graph = SerialFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst + logger.debug(f"serial: {time.time() - start_time: .2f}s -- {serial_graph.num_states()} nodes") + + start_time = time.time() + v_time_graph = vTimeFst(deterministic=deterministic).fst + v_ordinal_graph = vOrdinalFst(deterministic=deterministic) + v_date_graph = vDateFst(ordinal=v_ordinal_graph, deterministic=deterministic).fst + time_final = pynini.compose(time_graph, v_time_graph) + date_final = pynini.compose(date_graph, v_date_graph) + range_graph = RangeFst( + time=time_final, + date=date_final, + cardinal=cardinal, + deterministic=deterministic, + ).fst + logger.debug(f"range: {time.time() - start_time: .2f}s -- {range_graph.num_states()} nodes") + + # A quick fix to address money ranges: + # $150-$200 -> one hundred and fifty dollars to two hundred dollars + + dash = (pynutil.insert('name: "') + pynini.cross("-", "to") + pynutil.insert('"')).optimize() + + graph_range_money = pynini.closure( + money_graph + + pynutil.insert(" }") + + pynutil.insert(" tokens { ") + + dash + + pynutil.insert(" } ") + + pynutil.insert("tokens { ") + + money_graph, + 1, + ) + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electonic_graph, 1.11) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(range_graph, 1.1) + | pynutil.add_weight(serial_graph, 1.12) # should be higher than the rest of the classes + | pynutil.add_weight(graph_range_money, 1.1) + ) + + # roman_graph = RomanFst(deterministic=deterministic).fst + # classify |= pynutil.add_weight(roman_graph, 1.1) + + if not deterministic: + abbreviation_graph = AbbreviationFst(deterministic=deterministic).fst + classify |= pynutil.add_weight(abbreviation_graph, 100) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct) + | punct, + 1, + ) + + classify |= pynutil.add_weight(word_graph, 100) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py new file mode 100644 index 0000000000000000000000000000000000000000..19d4830bc2703e8ff58efaa466c63ff53e345e55 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py @@ -0,0 +1,227 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst as vCardinal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDate +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst as vDecimal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst as vElectronic +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst as vFraction +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst as vMeasure +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst as vMoney +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst as vRoman +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != 'None': + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, f"_{input_case}_en_tn_{deterministic}_deterministic{whitelist_file}_lm.far" + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + logger.info(f'ClassifyFst.fst was restored from {far_file}.') + else: + logger.info(f'Creating ClassifyFst grammars. This might take some time...') + # TAGGERS + cardinal = CardinalFst(deterministic=True, lm=True) + cardinal_tagger = cardinal + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=True) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=True) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=True, cardinal=cardinal) + fraction_graph = fraction.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=True) + measure_graph = measure.fst + date = DateFst(cardinal=cardinal, deterministic=True, lm=True) + date_graph = date.fst + punctuation = PunctuationFst(deterministic=True) + punct_graph = punctuation.graph + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).graph + time_graph = TimeFst(cardinal=cardinal, deterministic=True).fst + telephone_graph = TelephoneFst(deterministic=True).fst + electronic_graph = ElectronicFst(cardinal=cardinal, deterministic=True).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=False).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=False, input_file=whitelist) + whitelist_graph = whitelist.graph + serial_graph = SerialFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic, lm=True).fst + + # VERBALIZERS + cardinal = vCardinal(deterministic=True) + v_cardinal_graph = cardinal.fst + decimal = vDecimal(cardinal=cardinal, deterministic=True) + v_decimal_graph = decimal.fst + ordinal = vOrdinal(deterministic=True) + v_ordinal_graph = ordinal.fst + fraction = vFraction(deterministic=True, lm=True) + v_fraction_graph = fraction.fst + v_telephone_graph = vTelephone(deterministic=True).fst + v_electronic_graph = vElectronic(deterministic=True).fst + measure = vMeasure(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=False) + v_measure_graph = measure.fst + v_time_graph = vTime(deterministic=True).fst + v_date_graph = vDate(ordinal=ordinal, deterministic=deterministic, lm=True).fst + v_money_graph = vMoney(decimal=decimal, deterministic=deterministic).fst + v_roman_graph = vRoman(deterministic=deterministic).fst + v_word_graph = vWord(deterministic=deterministic).fst + + cardinal_or_date_final = plurals._priority_union(date_graph, cardinal_graph, NEMO_SIGMA) + cardinal_or_date_final = pynini.compose(cardinal_or_date_final, (v_cardinal_graph | v_date_graph)) + + time_final = pynini.compose(time_graph, v_time_graph) + ordinal_final = pynini.compose(ordinal_graph, v_ordinal_graph) + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(time_final, sem_w) + | pynutil.add_weight(pynini.compose(decimal_graph, v_decimal_graph), sem_w) + | pynutil.add_weight(pynini.compose(measure_graph, v_measure_graph), sem_w) + | pynutil.add_weight(ordinal_final, sem_w) + | pynutil.add_weight(pynini.compose(telephone_graph, v_telephone_graph), sem_w) + | pynutil.add_weight(pynini.compose(electronic_graph, v_electronic_graph), sem_w) + | pynutil.add_weight(pynini.compose(fraction_graph, v_fraction_graph), sem_w) + | pynutil.add_weight(pynini.compose(money_graph, v_money_graph), sem_w) + | pynutil.add_weight(cardinal_or_date_final, sem_w) + | pynutil.add_weight(whitelist_graph, sem_w) + | pynutil.add_weight( + pynini.compose(serial_graph, v_word_graph), 1.1001 + ) # should be higher than the rest of the classes + ).optimize() + + roman_graph = RomanFst(deterministic=deterministic, lm=True).fst + # the weight matches the word_graph weight for "I" cases in long sentences with multiple semiotic tokens + classify_and_verbalize |= pynutil.add_weight(pynini.compose(roman_graph, v_roman_graph), sem_w) + + date_final = pynini.compose(date_graph, v_date_graph) + range_graph = RangeFst( + time=time_final, cardinal=cardinal_tagger, date=date_final, deterministic=deterministic + ).fst + classify_and_verbalize |= pynutil.add_weight(pynini.compose(range_graph, v_word_graph), sem_w) + classify_and_verbalize = pynutil.insert("< ") + classify_and_verbalize + pynutil.insert(" >") + classify_and_verbalize |= pynutil.add_weight(word_graph, word_w) + + punct_only = pynutil.add_weight(punct_graph, weight=punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct_only), + 1, + ) + + def get_token_sem_graph(classify_and_verbalize): + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + + graph = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + return graph + + self.fst = get_token_sem_graph(classify_and_verbalize) + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..9eb06c435be4b45628dcaa3dd042b7c90665be61 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py @@ -0,0 +1,229 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.abbreviation import AbbreviationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst as vAbbreviation +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst as vCardinal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDate +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst as vDecimal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst as vElectronic +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst as vFraction +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst as vMeasure +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst as vMoney +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinal +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst as vRoman +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != 'None': + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, f"_{input_case}_en_tn_{deterministic}_deterministic{whitelist_file}.far" + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] + logger.info(f'ClassifyFst.fst was restored from {far_file}.') + else: + logger.info(f'Creating ClassifyFst grammars. This might take some time...') + # TAGGERS + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + deterministic_ordinal = OrdinalFst(cardinal=cardinal, deterministic=True) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) + fraction_graph = fraction.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + punctuation = PunctuationFst(deterministic=True) + punct_graph = punctuation.graph + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).graph + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electronic_graph = ElectronicFst(cardinal=cardinal, deterministic=deterministic).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = whitelist.graph + serial_graph = SerialFst(cardinal=cardinal, ordinal=deterministic_ordinal, deterministic=deterministic).fst + + # VERBALIZERS + cardinal = vCardinal(deterministic=deterministic) + v_cardinal_graph = cardinal.fst + decimal = vDecimal(cardinal=cardinal, deterministic=deterministic) + v_decimal_graph = decimal.fst + ordinal = vOrdinal(deterministic=deterministic) + v_ordinal_graph = ordinal.fst + fraction = vFraction(deterministic=deterministic) + v_fraction_graph = fraction.fst + v_telephone_graph = vTelephone(deterministic=deterministic).fst + v_electronic_graph = vElectronic(deterministic=deterministic).fst + measure = vMeasure(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + v_measure_graph = measure.fst + v_time_graph = vTime(deterministic=deterministic).fst + v_date_graph = vDate(ordinal=ordinal, deterministic=deterministic).fst + v_money_graph = vMoney(decimal=decimal, deterministic=deterministic).fst + v_roman_graph = vRoman(deterministic=deterministic).fst + v_abbreviation = vAbbreviation(deterministic=deterministic).fst + + det_v_time_graph = vTime(deterministic=True).fst + det_v_date_graph = vDate(ordinal=vOrdinal(deterministic=True), deterministic=True).fst + time_final = pynini.compose(time_graph, det_v_time_graph) + date_final = pynini.compose(date_graph, det_v_date_graph) + range_graph = RangeFst( + time=time_final, date=date_final, cardinal=CardinalFst(deterministic=True), deterministic=deterministic + ).fst + v_word_graph = vWord(deterministic=deterministic).fst + + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(whitelist_graph, sem_w) + | pynutil.add_weight(pynini.compose(time_graph, v_time_graph), sem_w) + | pynutil.add_weight(pynini.compose(decimal_graph, v_decimal_graph), sem_w) + | pynutil.add_weight(pynini.compose(measure_graph, v_measure_graph), sem_w) + | pynutil.add_weight(pynini.compose(cardinal_graph, v_cardinal_graph), sem_w) + | pynutil.add_weight(pynini.compose(ordinal_graph, v_ordinal_graph), sem_w) + | pynutil.add_weight(pynini.compose(telephone_graph, v_telephone_graph), sem_w) + | pynutil.add_weight(pynini.compose(electronic_graph, v_electronic_graph), sem_w) + | pynutil.add_weight(pynini.compose(fraction_graph, v_fraction_graph), sem_w) + | pynutil.add_weight(pynini.compose(money_graph, v_money_graph), sem_w) + | pynutil.add_weight(word_graph, word_w) + | pynutil.add_weight(pynini.compose(date_graph, v_date_graph), sem_w - 0.01) + | pynutil.add_weight(pynini.compose(range_graph, v_word_graph), sem_w) + | pynutil.add_weight( + pynini.compose(serial_graph, v_word_graph), 1.1001 + ) # should be higher than the rest of the classes + ).optimize() + + if not deterministic: + roman_graph = RomanFst(deterministic=deterministic).fst + # the weight matches the word_graph weight for "I" cases in long sentences with multiple semiotic tokens + classify_and_verbalize |= pynutil.add_weight(pynini.compose(roman_graph, v_roman_graph), word_w) + + abbreviation_graph = AbbreviationFst(whitelist=whitelist, deterministic=deterministic).fst + classify_and_verbalize |= pynutil.add_weight( + pynini.compose(abbreviation_graph, v_abbreviation), word_w + ) + + punct_only = pynutil.add_weight(punct_graph, weight=punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct_only), + 1, + ) + + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + + graph = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + self.fst = graph + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + + # to remove normalization options that still contain digits and some special symbols + # e.g., "P&E" -> {P and E, P&E}, "P & E" will be removed from the list of normalization options + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, pynini.union(NEMO_DIGIT, "&"))) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..2c87a20a3c03e58b7d8badd77e061e456da0856d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py @@ -0,0 +1,153 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_UPPER, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.roman import get_names +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + misses -> tokens { name: "mrs" } + for non-deterministic case: "Dr. Abc" -> + tokens { name: "drive" } tokens { name: "Abc" } + tokens { name: "doctor" } tokens { name: "Abc" } + tokens { name: "Dr." } tokens { name: "Abc" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): + whitelist = load_labels(file) + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y in whitelist] + else: + whitelist = [[x, y] for x, y in whitelist] + + if keep_punct_add_end: + whitelist.extend(augment_labels_with_punct_at_end(whitelist)) + + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/tts.tsv")) + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + _get_whitelist_graph(input_case, get_abs_path("data/whitelist/symbol.tsv")), + ).optimize() + + if deterministic: + names = get_names() + graph |= ( + pynini.cross(pynini.union("st", "St", "ST"), "Saint") + + pynini.closure(pynutil.delete(".")) + + pynini.accep(" ") + + names + ) + else: + graph |= _get_whitelist_graph( + input_case, get_abs_path("data/whitelist/alternatives.tsv"), keep_punct_add_end=True + ) + + for x in [".", ". "]: + graph |= ( + NEMO_UPPER + + pynini.closure(pynutil.delete(x) + NEMO_UPPER, 2) + + pynini.closure(pynutil.delete("."), 0, 1) + ) + + if not deterministic: + multiple_forms_whitelist_graph = get_formats(get_abs_path("data/whitelist/alternatives_all_format.tsv")) + graph |= multiple_forms_whitelist_graph + + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) | pynini.string_file( + get_abs_path("data/measure/unit_alternatives.tsv") + ) + graph_unit_plural = graph_unit @ SINGULAR_TO_PLURAL + units_graph = pynini.compose(NEMO_CHAR ** (3, ...), convert_space(graph_unit | graph_unit_plural)) + graph |= units_graph + + # convert to states only if comma is present before the abbreviation to avoid converting all caps words, + # e.g. "IN", "OH", "OK" + # TODO or only exclude above? + states = load_labels(get_abs_path("data/address/state.tsv")) + additional_options = [] + for x, y in states: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((x, f"{y[0]}.{y[1:]}")) + if not deterministic: + additional_options.append((x, f"{y[0]}.{y[1:]}.")) + + states.extend(additional_options) + state_graph = pynini.string_map(states) + graph |= pynini.closure(NEMO_NOT_SPACE, 1) + pynini.union(", ", ",") + pynini.invert(state_graph).optimize() + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = (convert_space(graph)).optimize() + + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() + + +def get_formats(input_f, input_case=INPUT_CASED, is_default=True): + """ + Adds various abbreviation format options to the list of acceptable input forms + """ + multiple_formats = load_labels(input_f) + additional_options = [] + for x, y in multiple_formats: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((f"{x}.", y)) # default "dr" -> doctor, this includes period "dr." -> doctor + additional_options.append((f"{x[0].upper() + x[1:]}", f"{y[0].upper() + y[1:]}")) # "Dr" -> Doctor + additional_options.append((f"{x[0].upper() + x[1:]}.", f"{y[0].upper() + y[1:]}")) # "Dr." -> Doctor + multiple_formats.extend(additional_options) + + if not is_default: + multiple_formats = [(x, f"|raw_start|{x}|raw_end||norm_start|{y}|norm_end|") for (x, y) in multiple_formats] + + multiple_formats = pynini.string_map(multiple_formats) + return multiple_formats diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..2823565b315f64bdf278d864add03297b25eed99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/taggers/word.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + get_abs_path, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. Considers sentence boundary exceptions. + e.g. sleep -> tokens { name: "sleep" } + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: GraphFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + punct = PunctuationFst().graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | NEMO_DIGIT).optimize() + graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # leave phones of format [HH AH0 L OW1] untouched + phoneme_unit = pynini.closure(NEMO_ALPHA, 1) + pynini.closure(NEMO_DIGIT) + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.accep(pynini.escape("]")) + ) + + # leave IPA phones of format [ˈdoʊv] untouched, single words and sentences with punctuation marks allowed + punct_marks = pynini.union(*punctuation.punct_marks).optimize() + stress = pynini.union("ˈ", "'", "ˌ") + ipa_phoneme_unit = pynini.string_file(get_abs_path("data/whitelist/ipa_symbols.tsv")) + # word in ipa form + ipa_phonemes = ( + pynini.closure(stress, 0, 1) + + pynini.closure(ipa_phoneme_unit, 1) + + pynini.closure(stress | ipa_phoneme_unit) + ) + # allow sentences of words in IPA format separated with spaces or punct marks + delim = (punct_marks | pynini.accep(" ")) ** (1, ...) + ipa_phonemes = ipa_phonemes + pynini.closure(delim + ipa_phonemes) + pynini.closure(delim, 0, 1) + ipa_phonemes = (pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]"))).optimize() + + if not deterministic: + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.accep(pynini.escape("]")) + ).optimize() + ipa_phonemes = ( + pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]")) + ).optimize() + + phoneme |= ipa_phonemes + self.graph = plurals._priority_union(convert_space(phoneme.optimize()), graph, NEMO_SIGMA) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a2a765a06cb6ddba97575cba8662f2498140f606 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py new file mode 100644 index 0000000000000000000000000000000000000000..87a2c93438027b4c67f9987abea7162aecd942b1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class AbbreviationFst(GraphFst): + """ + Finite state transducer for verbalizing abbreviations + e.g. tokens { abbreviation { value: "A B C" } } -> "ABC" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="abbreviation", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("value: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..485f1948c447e5d9927c98b90fc816265c2d776a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..07c3c35fb8002b5f40fcfda8b47e3b0f842e0bc4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py @@ -0,0 +1,107 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "february" day: "five" year: "twenty twelve" preserve_order: true } -> february fifth twenty twelve + date { day: "five" month: "february" year: "twenty twelve" preserve_order: true } -> the fifth of february twenty twelve + + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + phrase = pynini.closure(NEMO_NOT_QUOTE, 1) + day_cardinal = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = day_cardinal @ ordinal.suffix + period = pynutil.delete("text:") + delete_space + pynutil.delete("\"") + phrase + pynutil.delete("\"") + month = pynutil.delete("month:") + delete_space + pynutil.delete("\"") + phrase + pynutil.delete("\"") + + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + + # financial period + graph_fy = ( + pynutil.insert("the ") + period + pynutil.insert(" of ") + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # month (day) year + graph_mdy = ( + month + pynini.closure(delete_extra_space + day, 0, 1) + pynini.closure(delete_extra_space + year, 0, 1) + ) + # may 5 -> may five + if not deterministic and not lm: + graph_mdy |= ( + month + + pynini.closure(delete_extra_space + day_cardinal, 0, 1) + + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # day month year + graph_dmy = ( + pynutil.insert("the ") + + day + + delete_extra_space + + pynutil.insert("of ") + + month + + pynini.closure(delete_extra_space + year, 0, 1) + ) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = ( + (plurals._priority_union(graph_mdy, pynutil.add_weight(graph_dmy, 0.0001), NEMO_SIGMA) | year | graph_fy) + + delete_space + + optional_preserve_order + ) + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..7712e00bc61dcaa3246c62b013e4ea711eb6d0e7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { negative: "true" integer_part: "twelve" fractional_part: "five o o six" quantity: "billion" } -> minus twelve point five o o six billion + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + self.integer = pynutil.delete("integer_part:") + cardinal.integer + self.optional_integer = pynini.closure(self.integer + delete_space + insert_space, 0, 1) + self.fractional_default = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + self.fractional = pynutil.insert("point ") + self.fractional_default + + self.quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + graph = self.optional_sign + ( + self.integer + | (self.integer + self.quantity) + | (self.optional_integer + self.fractional + self.optional_quantity) + ) + + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + if not deterministic: + delete_tokens |= pynini.compose( + delete_tokens, + NEMO_SIGMA + + ( + pynini.cross(" point five", " and a half") + | pynini.cross("zero point five", "half") + | pynini.cross(" point two five", " and a quarter") + | pynini.cross("zero point two five", "quarter") + ), + ).optimize() + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe8ae3c477d8bdf36f78fd28ed9e7bfb5bfd583 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py @@ -0,0 +1,122 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_CHAR, + NEMO_LOWER, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() + graph_zero = pynini.cross("0", "zero") + long_numbers = pynutil.add_weight(graph_digit_no_zero + pynini.cross("000", " thousand"), MIN_NEG_WEIGHT) + + if not deterministic: + graph_zero |= pynini.cross("0", "o") | pynini.cross("0", "oh") + + graph_digit = graph_digit_no_zero | graph_zero + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + NEMO_NOT_BRACKET = pynini.difference(NEMO_CHAR, pynini.union("{", "}")).optimize() + dict_words = pynini.project(pynini.string_file(get_abs_path("data/electronic/words.tsv")), "output") + default_chars_symbols = pynini.cdrewrite( + pynutil.insert(" ") + (graph_symbols | graph_digit | long_numbers) + pynutil.insert(" "), + "", + "", + NEMO_SIGMA, + ) + default_chars_symbols = pynini.compose( + pynini.closure(NEMO_NOT_BRACKET), default_chars_symbols.optimize() + ).optimize() + + # this is far cases when user name was split by dictionary words, i.e. "sevicepart@ab.com" -> "service part" + space_separated_dict_words = pynutil.add_weight( + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_SPACE) + + NEMO_SPACE + + pynini.closure(NEMO_ALPHA | NEMO_SPACE), + MIN_NEG_WEIGHT, + ) + + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete("\"") + + (default_chars_symbols | space_separated_dict_words).optimize() + + pynutil.delete("\"") + ) + + domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + # this will be used for a safe fallback + domain_all = pynini.compose( + default_chars_symbols, + pynini.closure(TO_LOWER | NEMO_LOWER | NEMO_SPACE | pynutil.add_weight(dict_words, MIN_NEG_WEIGHT)), + ) + + domain = ( + domain_all + + insert_space + + plurals._priority_union( + domain_common, pynutil.add_weight(pynini.cross(".", "dot"), weight=0.0001), NEMO_SIGMA + ) + + pynini.closure(insert_space + default_chars_symbols, 0, 1) + ) + + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete("\"") + + (domain | pynutil.add_weight(domain_all, weight=100)).optimize() + + delete_space + + pynutil.delete("\"") + ).optimize() + + protocol = pynutil.delete("protocol: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = ( + pynini.closure(protocol + delete_space, 0, 1) + + pynini.closure(user_name + delete_space + pynutil.insert(" at ") + delete_space, 0, 1) + + domain + + delete_space + ).optimize() @ pynini.cdrewrite(delete_extra_space, "", "", NEMO_SIGMA) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..6358ae56041b4c69efe8bf09d7c7b44fcebcfcc1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } -> + twenty three and four fifth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + denominator_one = pynini.cross("denominator: \"one\"", "over one") + denominator_half = pynini.cross("denominator: \"two\"", "half") + denominator_quarter = pynini.cross("denominator: \"four\"", "quarter") + + denominator_rest = ( + pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) @ suffix + pynutil.delete("\"") + ) + + denominators = plurals._priority_union( + denominator_one, + plurals._priority_union( + denominator_half, + plurals._priority_union(denominator_quarter, denominator_rest, NEMO_SIGMA), + NEMO_SIGMA, + ), + NEMO_SIGMA, + ).optimize() + if not deterministic: + denominators |= pynutil.delete("denominator: \"") + (pynini.accep("four") @ suffix) + pynutil.delete("\"") + + numerator_one = pynutil.delete("numerator: \"") + pynini.accep("one") + pynutil.delete("\" ") + numerator_one = numerator_one + insert_space + denominators + numerator_rest = ( + pynutil.delete("numerator: \"") + + (pynini.closure(NEMO_NOT_QUOTE) - pynini.accep("one")) + + pynutil.delete("\" ") + ) + numerator_rest = numerator_rest + insert_space + denominators + numerator_rest @= pynini.cdrewrite( + plurals._priority_union(pynini.cross("half", "halves"), pynutil.insert("s"), NEMO_SIGMA), + "", + "[EOS]", + NEMO_SIGMA, + ) + + graph = numerator_one | numerator_rest + + conjunction = pynutil.insert("and ") + if not deterministic and not lm: + conjunction = pynini.closure(conjunction, 0, 1) + + integer = pynini.closure(integer + insert_space + conjunction, 0, 1) + + graph = integer + graph + graph @= pynini.cdrewrite( + pynini.cross("and one half", "and a half") | pynini.cross("over ones", "over one"), "", "[EOS]", NEMO_SIGMA + ) + + self.graph = graph + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..6668298e48a3d3ec1572afdf5f052a447473296d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py @@ -0,0 +1,122 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms + measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms + tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + optional_sign = cardinal.optional_sign + unit = ( + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + + pynutil.delete("\"") + + delete_space + ) + + if not deterministic: + unit |= pynini.compose(unit, pynini.cross(pynini.union("inch", "inches"), "\"")) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + + if not deterministic: + graph_decimal |= pynini.compose( + graph_decimal, + NEMO_SIGMA + + ( + pynini.cross(" point five", " and a half") + | pynini.cross("zero point five", "half") + | pynini.cross(" point two five", " and a quarter") + | pynini.cross("zero point two five", "quarter") + ), + ).optimize() + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_fraction = ( + pynutil.delete("fraction {") + delete_space + fraction.graph + delete_space + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal | graph_fraction) + delete_space + insert_space + unit + + # SH adds "preserve_order: true" by default + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + graph |= unit + insert_space + (graph_cardinal | graph_decimal) + delete_space + pynini.closure(preserve_order) + # for only unit + graph |= ( + pynutil.delete("cardinal { integer: \"-\"") + + delete_space + + pynutil.delete("}") + + delete_space + + unit + + pynini.closure(preserve_order) + ) + address = ( + pynutil.delete("units: \"address\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + math = ( + pynutil.delete("units: \"math\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + graph |= address | math + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..426a40fa8e31f4a691468e44f091cc927f576b43 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) + +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "twelve" fractional_part: "o five" currency: "dollars" } -> twelve o five dollars + + Args: + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + keep_space = pynini.accep(" ") + maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + min = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + + integer_part = decimal.integer + + # *** currency_maj + graph_integer = integer_part + keep_space + maj + + # *** currency_maj + (***) | ((and) *** current_min) + fractional = fractional_part + delete_extra_space + min + + if not deterministic: + fractional |= pynutil.insert("and ") + fractional + + graph_integer_with_minor = integer_part + keep_space + maj + keep_space + fractional + delete_preserve_order + + # *** point *** currency_maj + graph_decimal = decimal.numbers + keep_space + maj + + # *** current_min + graph_minor = fractional_part + delete_extra_space + min + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + if not deterministic: + graph |= graph_integer + delete_preserve_order + + per_units_normalized = pynini.project(per_units, "output") + remove_per_units_normalized = ( + pynutil.delete(' morphosyntactic_features: "') + + pynutil.insert(" ") + + per_units_normalized + + pynutil.delete('" ') + ) + graph += remove_per_units_normalized.ques + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..4a8efeb0822b4fa9f4bc2283dc52ffbb06076cc6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py @@ -0,0 +1,57 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "thirteen" } } -> thirteenth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/ordinal/digit.tsv")).invert() + graph_teens = pynini.string_file(get_abs_path("data/ordinal/teen.tsv")).invert() + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + convert_rest = pynutil.insert("th") + + suffix = pynini.cdrewrite( + graph_digit | graph_teens | pynini.cross("ty", "tieth") | convert_rest, + "", + "[EOS]", + NEMO_SIGMA, + ).optimize() + self.graph = pynini.compose(graph, suffix) + self.suffix = suffix + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..ea8a247730ea96fde94d5a9cf9668ff8e63f8742 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_CHAR, + NEMO_SIGMA, + NEMO_SPACE, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "en_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + punct_marks_all = PunctuationFst().punct_marks + + # no_space_before_punct assume no space before them + quotes = ["'", "\"", "``", "«"] + dashes = ["-", "—"] + brackets = ["<", "{", "(", r"\["] + open_close_single_quotes = [ + ("`", "`"), + ] + + open_close_double_quotes = [('"', '"'), ("``", "``"), ("“", "”")] + open_close_symbols = open_close_single_quotes + open_close_double_quotes + allow_space_before_punct = ["&"] + quotes + dashes + brackets + [k[0] for k in open_close_symbols] + + no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct] + no_space_before_punct = pynini.union(*no_space_before_punct) + no_space_after_punct = pynini.union(*brackets) + delete_space = pynutil.delete(" ") + delete_space_optional = pynini.closure(delete_space, 0, 1) + + # non_punct allows space + # delete space before no_space_before_punct marks, if present + non_punct = pynini.difference(NEMO_CHAR, no_space_before_punct).optimize() + graph = ( + pynini.closure(non_punct) + + pynini.closure( + no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT) + ) + + pynini.closure(non_punct) + ) + graph = pynini.closure(graph).optimize() + graph = pynini.compose( + graph, pynini.cdrewrite(pynini.cross("``", '"'), "", "", NEMO_SIGMA).optimize() + ).optimize() + + # remove space after no_space_after_punct (even if there are no matching closing brackets) + no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, NEMO_SIGMA, NEMO_SIGMA).optimize() + graph = pynini.compose(graph, no_space_after_punct).optimize() + + # remove space around text in quotes + single_quote = pynutil.add_weight(pynini.accep("`"), MIN_NEG_WEIGHT) + double_quotes = pynutil.add_weight(pynini.accep('"'), MIN_NEG_WEIGHT) + quotes_graph = ( + single_quote + delete_space_optional + NEMO_ALPHA + NEMO_SIGMA + delete_space_optional + single_quote + ).optimize() + + # this is to make sure multiple quotes are tagged from right to left without skipping any quotes in the left + not_alpha = pynini.difference(NEMO_CHAR, NEMO_ALPHA).optimize() | pynutil.add_weight( + NEMO_SPACE, MIN_NEG_WEIGHT + ) + end = pynini.closure(pynutil.add_weight(not_alpha, MIN_NEG_WEIGHT)) + quotes_graph |= ( + double_quotes + + delete_space_optional + + NEMO_ALPHA + + NEMO_SIGMA + + delete_space_optional + + double_quotes + + end + ) + + quotes_graph = pynutil.add_weight(quotes_graph, MIN_NEG_WEIGHT) + quotes_graph = NEMO_SIGMA + pynini.closure(NEMO_SIGMA + quotes_graph + NEMO_SIGMA) + + graph = pynini.compose(graph, quotes_graph).optimize() + + # remove space between a word and a single quote followed by s + remove_space_around_single_quote = pynini.cdrewrite( + delete_space_optional + pynini.union(*self.punct_marks["'"]) + delete_space, + NEMO_ALPHA, + pynini.union("s ", "s[EOS]"), + NEMO_SIGMA, + ) + + graph = pynini.compose(graph, remove_space_around_single_quote).optimize() + return graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py new file mode 100644 index 0000000000000000000000000000000000000000..2df6da1cf3ec7f1a325bba6c3662546712da98c5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing roman numerals + e.g. tokens { roman { integer: "one" } } -> one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + cardinal = pynini.closure(NEMO_NOT_QUOTE) + ordinal = pynini.compose(cardinal, suffix) + + graph = ( + pynutil.delete("key_cardinal: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" ") + + pynutil.delete("integer: \"") + + cardinal + + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("default_cardinal: \"default\" integer: \"") + cardinal + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("default_ordinal: \"default\" integer: \"") + ordinal + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("key_the_ordinal: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" ") + + pynutil.delete("integer: \"") + + pynini.closure(pynutil.insert("the "), 0, 1) + + ordinal + + pynutil.delete("\"") + ).optimize() + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..89052391932311b4a151c18cff797aa3a475a373 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + -> one, one two three, one two three, five six seven eight, one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..0f8a24c47dbeb54153647340d80707dd7c0736f2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py @@ -0,0 +1,103 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } -> twelve thirty a m e s t + time { hours: "twelve" } -> twelve o'clock + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + suffix = ( + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_suffix = pynini.closure(delete_space + insert_space + suffix, 0, 1) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph_hms = ( + hour + + pynutil.insert(" hours ") + + delete_space + + minute + + pynutil.insert(" minutes and ") + + delete_space + + second + + pynutil.insert(" seconds") + + optional_suffix + + optional_zone + ) + graph_hms @= pynini.cdrewrite(pynini.cross("one hours", "one hour"), "[BOS]", "", NEMO_SIGMA) + graph_hms @= pynini.cdrewrite( + pynutil.delete("o ") + | pynutil.add_weight(pynini.cross("o one minutes", "one minute"), -0.01) + | pynutil.add_weight(pynini.cross("o one seconds", "one second"), -0.01), + " ", + "", + NEMO_SIGMA, + ) + graph = hour + delete_space + insert_space + minute + optional_suffix + optional_zone + graph |= hour + insert_space + pynutil.insert("o'clock") + optional_zone + graph |= hour + delete_space + insert_space + suffix + optional_zone + graph |= graph_hms + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..fcb9c527d6651f96701bfa99e41118c74bbae752 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electronic_graph = ElectronicFst(deterministic=deterministic).fst + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + time_graph = TimeFst(deterministic=deterministic).fst + date_graph = DateFst(ordinal=ordinal, deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + time_graph + | date_graph + | money_graph + | measure_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | telephone_graph + | electronic_graph + | fraction_graph + | whitelist_graph + ) + + roman_graph = RomanFst(deterministic=deterministic).fst + graph |= roman_graph + + if not deterministic: + abbreviation_graph = AbbreviationFst(deterministic=deterministic).fst + graph |= abbreviation_graph + + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..823e1aad5ef053b24ad2e597483b08390958c4b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py @@ -0,0 +1,75 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..2553887c6bed3dcfc06244a94fc04b8b8406f0a7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "misses" } } -> misses + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..6be536ce5b5986755195fdea4a88147519752933 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing word + e.g. tokens { name: "sleep" } -> sleep + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/README.md b/zonos2/vendor/nemo_text_processing/text_normalization/es/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ec946a614da8661eb38cf96d7b4b2145fd367e2f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/README.md @@ -0,0 +1,22 @@ +# Localization Note + +Depending on locale, Spanish number strings will vary in formatting. In the EU and South American countries, it is common to use a period (".") or space to delineate groupings of three +digits. e.g. + `1.000.000` -> "un millón" + `1 000 000` -> "un millón" + +and commas (",") to seperate cardinal and decimal strings. e.g. + + `1,00` -> "uno coma cero cero" + +While Central and Northern America will use commas (",") to delineate groupings of three digits, e.g. + `1,000,000` -> "un millón" + +and periods (".") to seperate cardinal and decimal strings. e.g. + + `1.00` -> "uno coma cero cero" + +As inclusion of both forms will create inherrent ambiguity for verbalization, this module defaults to the former formatting (periods for cardinal delineation and commas for decimals). + +To toggle the alternate formatting, you may edit the `LOCALIZATION` variable in `nemo_text_processing.text_normalization.es.__init__` with the value of `'am'`. This will perform necessary +adjustments to all affected classes. \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bd1fc7d433efa6efa41a9182829300774b1c233d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCALIZATION = "eu" # Set to am for alternate formatting diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb4a9e73bcfb58181d1f6cdf030880093c031987 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv @@ -0,0 +1,27 @@ +LU lunes +lun. lunes +LUN lunes +ma. martes +MA martes +mar. martes +mi. miércoles +mié. miércoles +miér. miércoles +MIÉ miércoles +mié miércoles +miér miércoles +ju. jueves +JU jueves +jue jueves +JUE jueves +vie. viernes +vier. viernes +vie viernes +vier. viernes +sá. sábado +sáb sábado +sáb. sábado +SÁB sábado +do. domingo +dom. domingo +DOM domingo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7cfdf329aa5b2f13f94c650d7cbab5d5fe616ddd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 enero +2 febrero +3 marzo +4 abril +5 mayo +6 junio +7 julio +8 agosto +9 septiembre +10 octubre +11 noviembre +12 diciembre \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d08bba36c5432b1e69e9f8e68a078a2434e09e85 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv @@ -0,0 +1,44 @@ +ene. enero +en. enero +ENE enero +febr. febrero +FEB febrero +feb. febrero +mzo. marzo +mzo marzo +MAR marzo +mar. marzo +abr. abril +ABR abril +abr abril +my. mayo +MAY mayo +may. mayo +may mayo +jun. junio +JUN junio +jun junio +jul. julio +JUL julio +jul julio +AGO agosto +ag. agosto +agt. agosto +ago. agosto +set. septiembre +sept. septiembre +setbre. septiembre +sept septiembre +setbre septiembre +SEP septiembre +oct. octubre +OCT octubre +oct octubre +nov. noviembre +novbre. noviembre +NOV noviembre +nov noviembre +novbre noviembre +dic. diciembre +dicbre. diciembre +DIC diciembre diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..455b05d3e1b48388a88c8084c70a8de8af44094f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv @@ -0,0 +1,51 @@ +ane antes de nuestra era +a.n.e. antes de nuestra era +a. n. e. antes de nuestra era +aec antes de la era común +a.e.c antes de la era común +a. e. c. antes de la era común +ac antes de cristo +a.c. antes de cristo +a. c. antes de cristo +adc antes de cristo +a.d.c. antes de cristo +a. d. c. antes de cristo +a. de c. antes de cristo +ajc antes de jesucristo +a.j.c. antes de jesucristo +a. j. c. antes de jesucristo +adjc antes de jesucristo +a.d.j.c. antes de jesucristo +a. d. j. c. antes de jesucristo +a. de j.c. antes de jesucristo +a. de j. c. antes de jesucristo +ap antes del presente +a.p. antes del presente +a. p. antes del presente +bp antes del presente +b.p. antes del presente +b. p. antes del presente +ne nuestra era +n.e. nuestra era +n. e. nuestra era +ec era común +e.c. era común +e. c. era común +dc después de cristo +d.c. después de cristo +d. c. después de cristo +ddc después de cristo +d.d.c después de cristo +d. d. c. después de cristo +d. de c. después de cristo +ddjc después de jesucristo +d.d.j.c después de jesucristo +d. d. j. c. después de jesucristo +d. de j.c. después de jesucristo +d. de j. c. después de jesucristo +bc b c +b.c. b c +b. c. b c +ad a d +a.d. a d +a. d. a d diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2db92b2574167fb3436c079fd7941450ae0c316b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..06581ba1395c8606e04c1d82e2c56feafb19f186 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv @@ -0,0 +1,6 @@ +.com punto com +.gov punto gov +.net punto net +.in punto in +.ru punto ru +.gob punto gob diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a0c7562948684993372a2c389c9f32297550e5f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..77f5d50a4b9068cbb3b3056fc94f22ae227d316e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- guión +_ barra baja +! signo de exclamación +# almohadilla +$ signo de dólar +% signo de porcentaje +& et +' comilla +* asterisco ++ signo más +/ barra += signo igual +? signo de interrogación +^ acento circunflejo +` acento grave +{ llave izquierda +| barra vertical +} llave derecha +~ tilde +, coma +: dos puntos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ffebd210552da18e3be34229f81195342c978e7f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv @@ -0,0 +1,15 @@ +⅓ 1/3 +⅔ 2/3 +⅕ 1/5 +⅖ 2/5 +⅗ 3/5 +⅘ 4/5 +⅙ 1/6 +⅚ 5/6 +⅛ 1/8 +⅜ 3/8 +⅝ 5/8 +⅞ 7/8 +¼ 1/4 +½ 1/2 +¾ 3/4 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..32cc3930a0e6e77cddf3fdb01e9da87a63071fdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv @@ -0,0 +1,2 @@ +segundo medio +tercero tercio \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv new file mode 100644 index 0000000000000000000000000000000000000000..021dc0a101e03ba7815d43f4076a340af4dc37a7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv @@ -0,0 +1,9 @@ +millón millonésimo +millones millonésimo +billón billonésimo +billones billonésimo +trillón trillonésimo +trillones trillonésimo +mil millones mil millonésimo +mil billones mil billonésimo +mil trillones mil trillonésimo \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9e1a4ca4ce9f21ccef7f1d06a1bb068adb1f4561 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv @@ -0,0 +1,6 @@ +décimo décima +centésimo centésima +milésimo milésima +millonésimo millonésima +billonésimo billonésima +trillonésimo trillonésima \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a2a150c8939351d5f8786cc997cd639253a86f8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv @@ -0,0 +1,7 @@ ++ más +- menos +* por +× por +· por +/ entre +÷ entre diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2107c4691370f6121220b8f3b907ac16b685935c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv @@ -0,0 +1,18 @@ +cm centímetro +g gramo +h hora +kg kilogramo +km kilómetro +km² kilómetro cuadrado +l litro +m metro +m² metro cuadrado +m³ metro cúbico +ml mililitro +mm milímetro +ms milisegundo +min minuto +s segundo +° c grado centígrado +° f grado farenheit +° k grado kelvin diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8cc7dae9982fa7899278895ac7d20b8ee91f6399 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv @@ -0,0 +1,5 @@ +% por ciento +mph milla por hora +kph kilómetro por hora +gsm gramo por metro cuadrado +psi p s i diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..eb3ae2322e7a06e88c4abae5cf3cee6847ea1e10 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv @@ -0,0 +1,3 @@ +hora horas +milla por hora millas por hora +milla millas diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a7733c2558ad644245a0686f1d52d0530cf55b92 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv @@ -0,0 +1,21 @@ +centímetro centímetros +gramo gramos +kilogramo kilogramos +kilómetro kilómetros +kilómetro cuadrado kilómetros cuadrados +litro litros +metro metros +metro cuadrado metros cuadrados +metro cúbico metros cúbicos +mililitro mililitros +milímetro milímetros +milisegundo milisegundos +minuto minutos +segundo segundos +por ciento por ciento +kilómetro por hora kilómetros por hora +gramo por metro cuadrado gramos por metro cuadrado +p s i p s i +grado centígrado grados centígrados +grado farenheit grados farenheit +grado kelvin grados kelvin diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9dfd19ba570e5dac0cfafb0c1ce2e26677965997 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv @@ -0,0 +1,64 @@ +ars peso argentino +ar$ peso argentino +awg florín arubeño +bob boliviano +brl real brasileño +cad dólar canadiense +chf franco suizo +clp peso chileno +cny yuan chino +cop peso colombiano +crc colón costarricense +₡ colón +cup peso cubano +dop peso dominicano +rd$ peso dominicano +eur euro +gtq quetzal +gyd dólar guyanés +hkd dólar de hong kong +hnl lempira +htg gourde +inr rupia india +jpy yen +kr corona +kr. corona +krw won +mxn peso mexicano +nio córdoba +pab balboa +pen sol +s/ sol +pyg guaraní +₲ guaraní +rs rupia +rs. rupia +svc colón salvadoreño +ttd dólar de trinidad y tobago +usd dólar estadounidense +uyu peso uruguayo +uyw unidad previsional uruguaya +ves​ bolívar soberano +arp peso argentino +bop peso boliviano +cle escudo chileno +ecs sucre ecuatoriano +mxp peso mexicano +peh sol de oro +svc colón salvadoreño +veb bolívar venezolano +vef bolívar fuerte +€ euro +$ dólar +£ libra +₩ won +fr franco +mk marco +¥ yen +r$ real +₧ peseta +ptas peseta +us$ dólar estadounidense +mex$ peso mexicano +q quetzal +us $ dólar estadounidense diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..092b50202e49dd91cbb7ff71d0c24b7a28f6ce1d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv @@ -0,0 +1,215 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_major.tsv file. + +aed dírham +dh dírham +dhs dírham +afn afgani +all lek +amd dram armenio +ang florín antillano neerlandés +aoa kwanza +aok primer kwanza +aon nuevo kwanza +aor kwanza reajustado +azn manat azerbaiyano +bam marco convertible +bbd dólar barbadense +bdt taka +bgn leva búlgara +bhd dinar bareiní +bif franco de burundi +bmd dólar bermudeño +bd$ dólar bermudeño +bnd dólar de brunéi +bsd dólar bahameño +b$ dólar bahameño +btn ngultrum +bwp pula +byn rublo bielorruso +bzd dólar beliceño +bz$ dólar beliceño +cdf franco congoleño +cou unidad de valor real +cve escudo caboverdiano +czk corona checa +djf franco yibutiano +dkk corona danesa +dzd dinar argelino +egp libra egipcia +ern nakfa +etb birr etíope +fjd dólar fiyiano +fj$ dólar fiyiano +fkp libra malvinense +gbp libra esterlina +gel lari +ghs cedi ghanés +gip libra de gibraltar +gmd dalasi +gnf franco guineano +hrk kuna +huf forinto +idr rupia indonesia +ils nuevo séquel israelí +iqd dinar iraquí +irr rial iraní +isk corona islandesa +jmd dólar jamaiquino +jod dinar jordano +kes chelín keniano +kgs som +khr riel +kmf franco comorense +kpw won norcoreano +kwd dinar kuwaití +kyd dólar de las islas caimán +kzt tenge +lak kip +lbp libra libanesa +lkr rupia de sri lanka +lrd dólar liberiano +lsl loti +lyd dinar libio +mad dírham marroquí +mdl leu moldavo +mga ariary malgache +mkd denar +mmk kyat +mnt tugrik +₮ tugrik +mop pataca +mop$ pataca +mru uguiya +mur rupia de mauricio +mvr rufiyaa +mwk kwacha +myr ringgit malayo +mzn metical mozambiqueño +nad dólar namibio +ngn naira +₦ naira +nok corona noruega +npr rupia nepalí +nzd dólar neozelandés +nz$ dólar neozelandés +omr rial omaní +pgk kina +php peso filipino +₱ peso filipino +P peso filipino +pkr rupia pakistaní +pln esloti +zł esloti +zł. esloti +qar rial catarí +ron leu rumano +rsd dinar serbio +rub rublo ruso +rwf franco ruandés +sar rial saudí +sbd dólar de las islas salomón +scr rupia seychelense +sdg libra sudanesa +sek corona sueca +shp libra de santa elena +sll leone +sos chelín somalí +srd dólar surinamés +ssp libra sursudanesa +stn dobra +syp libra siria +szl lilangeni +thb baht +฿ baht +tjs somoni tayiko +tmt manat turcomano +tnd dinar tunecino +top paʻanga +try lira turca +twd nuevo dólar taiwanés +tzs chelín tanzano +uah grivna +ugx chelín ugandés +uzs som uzbeko +vnd dong vietnamita +₫ dong vietnamita +vuv vatu +wst tala +xaf franco c f a de áfrica central +xof franco c f a de áfrica occidental +xpf franco c f p +xsu sistema unitario de compensación regional +xua unidad de cuenta del banco africano de desarrollo +yer rial yemení +zar rand +zmw kwacha zambiano +zwl dólar zimbabuense +adf franco andorrano +adp peseta andorrana +afa afgani +aon kwanza angoleño +aor kwanza angoleño reajustado +ats chelín austríaco +bec franco belga +bef franco belga +bel franco belga +bgl lev búlgaro +csd dinar serbio +csk corona checoslovaca +cyp libra chipriota +ddm marco de rda +dem marco alemán +eek corona estonia +ehp peseta saharaui +fim marco finlandés +frf franco francés +grd dracma griega +iep libra irlandesa +itl lira italiana +luf franco luxemburgués +mtl lira maltesa +nlg florín neerlandés +pei inti +plz esloti polaco +pte escudo portugués +rur rublo ruso +sit tólar esloveno +skk corona eslovaca +srg guilder surinamés +sur rublo de la unión soviética +trl lira turca +yud nuevo dinar yugoslavo +yum dinar yugoslavo +zal rand surafricano código de fondos +zrn nuevo zaire +zrz zaire +zml kwacha zambiano +kč corona checa +aud dólar australiano +bov boliviano con mantenimiento de valor respecto al dólar estadounidense +che euro wir +chw franco wir +clf unidad de fomento +cuc peso convertible +mxv unidad de inversión mexicana +sgd dólar de singapur +usn dólar estadounidense siguiente día +uyi peso uruguayo en unidades indexadas +xag plata +xau oro +xcd dólar del caribe oriental +xdr derecho especial de giro +xpd paladio +xpt platino +ara austral argentino +arc peso moneda corriente argentino +arf peso fuerte argentino +arg peso oro sellado argentino +arl peso ley argentino +arm peso moneda nacional argentino +ecv unidad de valor constante ecuatoriana +esa peseta española +esb peseta española +esp peseta española +xeu unidad monetaria europea diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f61f345f60dd7ade187b65fd14e2ef00d91a1ca9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv @@ -0,0 +1,64 @@ +ars centavo +ar$ centavo +awg centavo +bob centavo +brl centavo +cad centavo +chf céntimo +clp centavo +cny jiao +cop centavo +crc céntimo +₡ céntimo +cup centavo +dop centavo +rd$ centavo +eur centavo +gtq centavo +gyd centavo +hkd centavo +hnl centavo +htg céntimo +inr paisa +jpy centavo +kr øre +kr. øre +krw chon +mxn centavo +nio centavo +pab centésimo +pen céntimo +s/ céntimo +pyg céntimo +₲ céntimo +rs paisa +rs. paisa +svc centavo +ttd céntimo +usd centavo +uyu centésimo +uyw centésimo +ves​ céntimo +arp centavo +bop centavo +cle centésimo +ecs centésimo +mxp centavo +peh centavo +svc centavo +veb céntimo +vef céntimo +€ céntimo +$ centavo +£ penique +₩ chon +fr céntimo +mk céntimo +¥ centavo +r$ centavo +₧ céntimo +ptas céntimo +us$ centavo +mex$ centavo +q centavo +us $ centavo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..80cda16e94f12d19b34c93b99bbff788baa50b74 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv @@ -0,0 +1,181 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_minor.tsv file. + +aed fils +dh fils +dhs fils +afn pul +all qindarke +amd luma +ang centavo +aoa céntimo +aok lwei +aon lwei +aor lwei +aud centavo +azn qəpik +bam feninga +bbd centavo +bdt poisha +bgn stotinka +bhd fils +bif céntimo +bmd centavo +bd$ centavo +bnd centavo +bov centavo con mantenimiento de valor respecto al Dólar estadounidense +r$ centavo +bsd centavo +b$ centavo +btn chertum +bwp thebe +byn kap +bzd centavo +bz$ centavo +cdf céntimo +che centavo +chw céntimo +cuc centavo +cve centavo +czk haléř +djf céntimo +dkk øre +dzd céntimo +egp piastra +ern céntimo +etb santim +fjd centavo +fj$ centavo +fkp penique +gbp penique +gel tetri +ghs pesewa +gip penique +gmd butut +gnf céntimo +hrk lipa +huf krajczár +idr sen +ils agorot +iqd fils +irr dinar +isk aurar +jmd céntimo +jod qirsh +kes centavo +kgs tiyin +khr sen +kmf céntimo +kpw chon +kwd fils +kyd centavo +kzt tiyin +lak att +lbp piastra +lkr céntimo +lrd centavo +lsl sente +mad céntimo +mdl ban +mkd deni +mmk pya +mnt mongo +₮ mongo +mop avo +mop$ avo +mru jum +mur centavo +mvr laari +mwk támbala +myr sen +nad centavo +ngn kobo +₦ kobo +nok øre +npr paisa +nzd centavo +nz$ centavo +omr baisa +pgk toea +php céntimo +₱ céntimo +P céntimo +pkr paisa +pln groszy +zł groszy +zł. groszy +qar dirham +ron ban +rsd para +rub kopek +rwf céntimo +sar halalá +sbd centavo +scr centavo +sdg piastra +sek öre +sgd centavo +shp penique +sos céntimo +srd centavo +ssp piastra +stn céntimo +syp piastra +szl centavo +thb satang +฿ satang +tjs diram +tmt teňňe +tnd millime +top seniti +try kuruş +tzs céntimo +uah kopek +ugx centavo +usn centavo +uyi centésimo +uzs tiyin +vnd xu +₫ xu +wst sene +xcd centavo +xpf céntimo +yer fils +zmw ngwee +zwl centavo +adf céntimo +adp céntimo +afa pul +ara centavo +arc centavo +arf centavo +arg centavo +arl centavo +arm centavo +bec céntimo +bef céntimo +bel céntimo +bgl stotinki +csd para +csk heller +cyp céntimo +ddm pfennig +dem pfennig +eek céntimo +esa céntimo +esb céntimo +esp céntimo +frf céntimo +iep penique +luf céntimo +mtl céntimo +nlg céntimo +pei céntimo +plz groszy +rur kopek +sit stotin +skk heller +sur kopek +trl kuruş +zml ngwee +kč haléř diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c1d94fd87be3fa2e4034be40f90b84b658f439b3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv @@ -0,0 +1,8 @@ +gourde gourdes +rupia india rupias indias +corona coronas +córdoba córdobas +rupia rupias +unidad previsional uruguaya unidades previsionales uruguayas +libra libras +peseta pesetas diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3d7c27559d78164e672bbe848d509ce412de2163 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv @@ -0,0 +1,56 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_plural_fem.tsv file. + +leva búlgara levas búlgaras +pula pula +unidad de fomento unidades de fomento +unidad de valor real unidades de valor real +corona checa coronas checas +corona danesa coronas danesas +libra egipcia libras egipcias +libra malvinense libras malvinenses +libra esterlina libras esterlinas +libra de gibraltar libras de gibraltar +kuna kuna +rupia indonesia rupias indonesias +corona islandesa coronas islandesas +libra libanesa libras libanesas +rupia de sri lanka rupias de sri lanka +pataca patacas +uguiya uguiya +rupia de mauricio rupias de mauricio +unidad de inversión mexicana unidades de inversión mexicana +rufiyaa rufiyaa +corona noruega coronas noruegas +rupia nepalí rupias nepalíes +rupia pakistaní rupias pakistaníes +rupia seychelense rupias seychelenses +libra sudanesa libras sudanesas +corona sueca coronas suecas +libra de santa elena libras de santa elena +libra siria libras sirias +lira turca liras turcas +grivna grivnas +unidad de cuenta del banco africano de desarrollo unidades de cuenta del banco africano de desarrollo +peseta andorrana pesetas andorranas +corona checoslovaca coronas checoslovacas +libra chipriota libras chipriotas +unidad de valor constante ecuatoriana unidades de valor constante ecuatorianas +corona estonia coronas estonias +peseta saharaui pesetas saharauies +peseta española pesetas españolas +dracma griega dracmas griegas +libra irlandesa libras irlandesas +lira italiana liras italianas +lira maltesa liras maltesas +corona eslovaca coronas eslovacas +lira turca liras turcas +unidad monetaria europea unidades monetaria europea +feninga feninga +poisha poisha +stotinka stotinki +piastra piastras +pesewa pesewas +pya pyas +támbala támbala +corona checa coronas checas diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..76954cc0d5614f6a8bdd695264efef45dbb44e4a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv @@ -0,0 +1,51 @@ +real brasileño reales brasileños +peso argentino pesos argentinos +florín arubeño florines arubeños +boliviano bolivianos +dólar canadiense dólares canadienses +franco suizo francos suizos +peso chileno pesos chilenos +yuan chino yuan chinos +peso colombiano pesos colombianos +colón costarricense colones costarricenses +peso cubano pesos cubanos +peso dominicano pesos dominicanos +euro euros +quetzal quetzales +dólar guyanés dólares guyaneses +dólar de hong kong dólares de hong kong +lempira lempiras +yen yenes +won wones +balboa balboas +sol soles +guaraní guaraníes +dólar de trinidad y tobago dólares de trinidad y tobago +dólar estadounidense dólares estadounidenses +peso uruguayo pesos uruguayos +bolívar soberano bolívares soberanos +peso argentino pesos argentinos +peso boliviano pesos bolivianos +escudo chileno escudos chilenos +sucre ecuatoriano sucres ecuatorianos +peso mexicano pesos mexicanos +sol de oro soles de oro +colón salvadoreño colones salvadoreños +bolívar venezolano bolívares venezolanos +bolívar fuerte bolívares fuertes +centavo centavos +céntimo céntimos +øre øre +penique peniques +paisa paise +chon chon +øre øre +centésimo centésimos +dólar dólares +won wones +franco francos +marco marcos +yen yenes +real reales +colón colones +quetzal quetzales diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..548558b3463bb06e5105186204da1234367ca366 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv @@ -0,0 +1,199 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_plural_masc.tsv file. + +dírham dírhams +afgani afganis +lek lekë +dram armenio dram armenios +florín antillano neerlandés florines antillanos neerlandeses +kwanza kwanzas +primer kwanza primeros kwanzas +nuevo kwanza nuevos kwanzas +kwanza reajustado kwanzas reajustados +dólar australiano dólares australianos +manat azerbaiyano manat azerbaiyanos +marco convertible marcos convertibles +dólar barbadense dólares barbadenses +taka taka +dinar bareiní dinares bareinís +franco de burundi francos de burundi +dólar bermudeño dólares bermudeños +dólar de brunéi dólares de brunéi +boliviano con mantenimiento de valor respecto al dólar estadounidense bolivianos con mantenimiento de valor respecto al dólar estadounidense +dólar bahameño dólares bahameños +ngultrum ngultrum +rublo bielorruso rublos bielorrusos +dólar beliceño dólares beliceños +franco congoleño francos congoleños +euro wir euros wir +franco wir francos wir +peso convertible pesos convertibles +escudo caboverdiano escudos caboverdianos +franco yibutiano francos yibutianos +dinar argelino dinares argelinos +nakfa nakfa +birr etíope birr etíopes +dólar fiyiano dólares fiyianos +lari lari +cedi ghanés cedis ghaneses +dalasi dalasis +franco guineano francos guineanos +forinto forintos +nuevo séquel israelí nuevos séqueles israelíes +dinar iraquí dinares iraquíes +rial iraní riales iraníes +dólar jamaiquino dólares jamaiquinos +dinar jordano dinares jordanos +chelín keniano chelines kenianos +som som +riel riel +franco comorense francos comorenses +won norcoreano wones norcoreanos +dinar kuwaití dinares kuwaitíes +dólar de las islas caimán dólares de las islas caimán +tenge tenge +kip kip +dólar liberiano dólares liberianos +loti maloti +dinar libio dinares libios +dírham marroquí dírhams marroquíes +leu moldavo lei moldavos +ariary malgache ariary +denar denares +kyat kyats +tugrik tugrik +kwacha kwacha +ringgit malayo ringgit malayos +metical mozambiqueño meticales mozambiqueños +dólar namibio dólares namibios +naira naira +dólar neozelandés dólares neozelandeses +rial omaní riales omaníes +kina kina +peso filipino pesos filipinos +esloti eslotis +rial catarí riales cataríes +leu rumano lei rumanos +dinar serbio dinares serbios +rublo ruso rublos rusos +franco ruandés francos ruandeses +rial saudí riales saudíes +dólar de las islas salomón dólares de las islas salomón +dólar de singapur dólares de singapur +leone leones +chelín somalí chelines somalíes +dólar surinamés dólares surinameses +libra sursudanesa libras sursudanesas +dobra dobras +lilangeni emalangeni +baht baht +somoni tayiko somoni tayikos +manat turcomano manat turcomanos +dinar tunecino dinares tunecinos +paʻanga paʻanga +nuevo dólar taiwanés nuevos dólares taiwaneses +chelín tanzano chelines tanzanos +chelín ugandés chelines ugandeses +dólar estadounidense siguiente día dólares estadounidenses siguiente día +peso uruguayo en unidades indexadas pesos uruguayos en unidades indexadas +som uzbeko som uzbekos +dong vietnamita dong vietnamitas +vatu vatu +tala tala +franco c f a de áfrica central francos c f a de áfrica central +plata plata +oro oro +dólar del caribe oriental dólares del caribe oriental +derecho especial de giro derechos especiales de giro +franco c f a de áfrica occidental francos c f a de áfrica occidental +paladio paladio +franco c f p francos c f p +platino platino +sistema unitario de compensación regional sistemas unitarios de compensación regional +rial yemení riales yemeníes +rand rand +kwacha zambiano kwacha zambianos +dólar zimbabuense dólares zimbabuenses +franco andorrano francos andorranos +kwanza angoleño kwanzas angoleños +kwanza angoleño reajustado kwanzas angoleños reajustados +austral argentino australes argentinos +peso moneda corriente argentino pesos moneda corriente argentinos +peso fuerte argentino pesos fuertes argentinos +peso oro sellado argentino pesos oro sellado argentinos +peso ley argentino pesos ley argentinos +peso moneda nacional argentino pesos moneda nacional argentinos +chelín austríaco chelines austríacos +franco belga francos belgas +lev búlgaro lev búlgaros +dinar serbio dinares serbios +marco de rda marcos de rda +marco alemán marcos alemanes +marco finlandés marcos finlandeses +franco francés francos franceses +franco luxemburgués francos luxemburgueses +florín neerlandés florines neerlandeses +inti intis +esloti polaco eslotis polacos +escudo portugués escudos portugueses +rublo ruso rublos rusos +tólar esloveno tólares eslovenos +guilder surinamés guilder surinameses +rublo de la unión soviética rublos de la unión soviética +nuevo dinar yugoslavo nuevos dinares yugoslavos +dinar yugoslavo dinares yugoslavos +rand surafricano código de fondos rands surafricanos código de fondos +nuevo zaire nuevos zaires +zaire zaires +kwacha zambiano kwacha zambianos +fils fils +pul pul +qindarke qindarka +luma luma +lwei lwei +qəpik qəpik +centavo con mantenimiento de valor respecto al dólar estadounidense centavos con mantenimiento de valor respecto al dólar estadounidense +chertum chertums +thebe thebe +kap kopeks +haléř haléř +santim santim +tetri tetri +butut bututs +lipa lipa +krajczár krajczár +sen sen +agorot agorot +dinar dinares +aurar aurar +qirsh qirsh +tiyin tiyin +att att +sente lisente +ban bani +deni deni +mongo mongo +avo avos +jum jums +laari laari +kobo kobo +baisa baise +toea toea +groszy groszy +dirham dirhams +öre öre +diram dirams +pfennig pfennigs +kopek kopeks +para para +halalá halalás +satang satang +teňňe teňňe +millime millimes +seniti seniti +kuruş kuruş +xu xu +sene sene +ngwee ngwee +heller hellers +stotin stotinov diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..462803637ed571078f22949c75a146655ceff0ad --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +un 1 +dos 2 +tres 3 +cuatro 4 +cinco 5 +seis 6 +siete 7 +ocho 8 +nueve 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bf651cd0a9786151f7b26e02a2f5ba09b0c5c3f8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +doscientos 2 +trescientos 3 +cuatrocientos 4 +quinientos 5 +seiscientos 6 +setecientos 7 +ochocientos 8 +novecientos 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..58bce3e5dcff0019538bb75996b0822c7827fbf0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv @@ -0,0 +1,6 @@ +millón +millones +billón +billones +trillón +trillones \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..370526ea57c3edce5fd6e7e4f8647a1715dc76e4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +diez 10 +once 11 +doce 12 +trece 13 +catorce 14 +quince 15 +dieciséis 16 +diecisiete 17 +dieciocho 18 +diecinueve 19 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..51b21081b3e35e10d6efbc87b4b274e20e183af8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv @@ -0,0 +1,7 @@ +treinta 3 +cuarenta 4 +cincuenta 5 +sesenta 6 +setenta 7 +ochenta 8 +noventa 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c378a3ad9817d72e60c7b9a762f6c5a3dd804098 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv @@ -0,0 +1,10 @@ +veinte 20 +veintiún 21 +veintidós 22 +veintitrés 23 +veinticuatro 24 +veinticinco 25 +veintiséis 26 +veintisiete 27 +veintiocho 28 +veintinueve 29 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..945785283a516d40edc75a693777ec6ef4a72c01 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv @@ -0,0 +1 @@ +cero 0 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..08d86ac928b4ce0bb7220acba13173af6f998816 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv @@ -0,0 +1,11 @@ +primero uno +primera una +primero un +segundo dos +tercero tres +cuarto cuatro +quinto cinco +sexto seis +séptimo siete +octavo ocho +noveno nueve \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..10eabb11de3ca51a0fbc508a8b88129d626b0e7a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv @@ -0,0 +1,20 @@ +ᵉʳ apocope +er apocope +º gender_masc +° gender_masc +ro gender_masc +ero gender_masc +do gender_masc +o gender_masc +to gender_masc +mo gender_masc +vo gender_masc +ª gender_fem +ra gender_fem +era gender_fem +a gender_fem +da gender_fem +ta gender_fem +ma gender_fem +va gender_fem +na gender_fem diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..65867fd84ecacd446bc23ba1572c9c921d1d186c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv @@ -0,0 +1,18 @@ +centésimo ciento +centésimo cien +ducentésimo doscientos +ducentésima doscientas +tricentésimo trescientos +tricentésima trescientas +cuadringentésimo cuatrocientos +cuadringentésima cuatrocientas +quingentésimo quinientos +quingentésima quinientas +sexcentésimo seiscientos +sexcentésima seiscientas +septingentésimo setecientos +septingentésima setecientas +octingentésimo ochocientos +octingentésima ochocientas +noningentésimo novecientos +noningentésima novecientas \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..688da504d26484ca584567da6c63e96781ac74c9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv @@ -0,0 +1,10 @@ +di +mi +vi +i +v +x +l +c +d +m \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1ffcfb2697d45d0a583925ee927bdd2b5b775a62 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv @@ -0,0 +1,10 @@ +décimo diez +undécimo once +duodécimo doce +decimotercero trece +decimocuarto catorce +decimoquinto quince +decimosexto dieciséis +decimoséptimo diecisiete +decimooctavo dieciocho +decimonoveno diecinueve \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..21c488ef2ab870a98d54119013cb068b56d66d35 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv @@ -0,0 +1,8 @@ +vigésimo veinte +trigésimo treinta +cuadragésimo cuarenta +quincuagésimo cincuenta +sexagésimo sesenta +septuagésimo setenta +octogésimo ochenta +nonagésimo noventa \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8fd7634438a375c10b86d4e0681e8e82a71a044f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv @@ -0,0 +1,11 @@ +vigesimoprimero veintiuno +vigesimoprimera veintiuna +vigesimoprimero veintiún +vigesimosegundo veintidós +vigesimotercero veintitrés +vigesimocuarto veinticuatro +vigesimoquinto veinticinco +vigesimosexto veintiséis +vigesimoséptimo veintisiete +vigesimoctavo veintiocho +vigesimonoveno veintinueve diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2db92b2574167fb3436c079fd7941450ae0c316b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0610b4a54ff3c9c90ad16a4ae9a9428a2a5484f4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv @@ -0,0 +1,9 @@ +i 1 +ii 2 +iii 3 +iv 4 +v 5 +vi 6 +vii 7 +viii 8 +ix 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..cdbdb68147261048298134373f77c3ade2689659 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv @@ -0,0 +1,9 @@ +c 1 +cc 2 +ccc 3 +cd 4 +d 5 +dc 6 +dcc 7 +dccc 8 +cm 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv new file mode 100644 index 0000000000000000000000000000000000000000..19e96b9c6decb7c9760ad34e21a9602bc6393d7d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv @@ -0,0 +1,3 @@ +m 1 +mm 2 +mmm 3 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ac043aa146bfeb1600a76b37ec59ef5254dc60b8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv @@ -0,0 +1,9 @@ +x 1 +xx 2 +xxx 3 +xl 4 +l 5 +lx 6 +lxx 7 +lxxx 8 +xc 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fc226eb30488da403e43c8b43dffa434b06805a7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv @@ -0,0 +1,2 @@ +la dirección de ip es la dirección de i p es +el ip es el i p es diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..33efe4550791ae50613088c4696813e263a23281 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv @@ -0,0 +1,10 @@ +llámame al +llámeme al +llamar al +llama al +llame al +marque +mi número es +mi teléfono es +mi número de teléfono es +el número de teléfono es diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0c266580597bdc279fe0ad53c60d7e9f7c866484 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv @@ -0,0 +1,6 @@ +una +dos +tres +cuatro +cinco +seis diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6bdff30993d42bdc60725e9a507a83d3df537213 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv @@ -0,0 +1,2 @@ +quince cuarto +treinta media \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d10160d0673f43006e0151c641fd7ff7b024e914 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv @@ -0,0 +1,5 @@ +siete +ocho +nueve +diez +once diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e681afc6c25c71eef187fc7a300df91cb411149b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv @@ -0,0 +1,12 @@ +1 12 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv new file mode 100644 index 0000000000000000000000000000000000000000..13f43dc5b9b94144846e164611b8391010b94c22 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv @@ -0,0 +1,25 @@ +1 0 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 +13 12 +14 13 +15 14 +16 15 +17 16 +18 17 +19 18 +20 19 +21 20 +22 21 +23 22 +0 23 +1 24 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b7765ac2a3c6158197b46e5886345e0e08b6b3ac --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv @@ -0,0 +1,13 @@ +12 0 +1 13 +2 14 +3 15 +4 16 +5 17 +6 18 +7 19 +8 20 +9 21 +10 22 +11 23 +12 24 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv new file mode 100644 index 0000000000000000000000000000000000000000..edab4d5b03d6fd6c8187ea76523f61f0df3410b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv @@ -0,0 +1,59 @@ +1 59 +2 58 +3 57 +4 56 +5 55 +6 54 +7 53 +8 52 +9 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 9 +52 8 +53 7 +54 6 +55 5 +56 4 +57 3 +58 2 +59 1 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e329184d2bd0c46ed5c8fa10609b77a1c553197c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv @@ -0,0 +1,12 @@ +un +una +dos +tres +cuatro +cinco +seis +siete +ocho +nueve +diez +once \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9ba6b1ceb1704b78d07af0f3c5567bbebe5f8b34 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv @@ -0,0 +1,20 @@ +p.m. pm +p. m. pm +p.m pm +p m. pm +pm pm +P.M. pm +P.M pm +P M. pm +P. M. pm +PM pm +a.m. am +a.m. am +a.m am +a m. am +am am +A.M. am +A.M am +A M. am +A. M. am +AM am \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6c86a3e4e40c94dbdf734d4599f6396f648fdf39 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv @@ -0,0 +1,42 @@ +utc u t c +cst c s t +cet c e t +pst p s t +est e s t +mdt m d t +mst m s t +pt p t +et e t +mt m t +gmt g m t +adt hora de verano del atlántico +amt hora estándar del amazonas +art hora estándar de argentina +ast hora estándar del atlántico +bot hora de bolivia +brt hora estándar de brasilia +clst hora de verano de chile +clt hora estándar de chile +cot hora estándar de colombia +east hora estándar de la isla de pascua +ect hora de ecuador +eeast hora de verano de la isla de pascua +eest hora de verano de europa oriental +eet hora estándar de europa oriental +fkst hora de verano de las malvinas +fnt hora estándar de fernando de noronha +galt hora de galápagos +gft hora de la guayana francesa +gyt hora de guyana +hkt hora estándar de hong kong +jst hora estándar de japón +kst hora estándar de corea +pet hora estándar de perú +pyst hora de verano de paraguay +pyt hora estándar de paraguay +sgt hora de singapur +uyst hora de verano de uruguay +uyt hora de uruguay +vet hora de venezuela +west hora de verano de europa oriental +wet hora estándar de europa oriental diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7f94b24b7205549e279303188871b6dba1f620c9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv @@ -0,0 +1,105 @@ +# Additional time zones are listed below. +# To use them, simply copy them to the main time_zone.tsv file. + +acdt hora de verano de australia central +acst hora estándar de australia central +act hora de acre +acwst hora estándar de australia centroccidental +aedt hora de verano de australia oriental +aest hora estándar de australia oriental +aft hora de afganistán +akdt tiempo de verano de alaska +akst tiempo estándar de alaska +almt hora de almá-atá +aqtt hora de aktobé +awst hora estándar de australia occidental +azost hora de verano de las azores +azot hora estándar de las azores +azt hora estándar de azerbaiyán +bdt hora de brunéi +bnt hora de brunéi darussalam +bst hora de verano británica +btt hora de bután +cast hora de casey +cat hora de áfrica central +cct hora de las islas cocos +cest hora de verano de europa central +chut hora de chuuk +cvt hora estándar de cabo verde +cxt hora de la isla de navidad +chst hora estándar de chamorro +eat hora de áfrica oriental +egst hora de verano de groenlandia oriental +egt hora estándar de groenlandia oriental +fjst hora de verano de fiyi +fjt hora estándar de fiyi +gamt hora de gambier +ict hora de indochina +idt hora de verano de israel +iot hora del océano índico +irdt hora de verano de irán +irst hora estándar de irán +ist i s t +kgt hora de kirguistán +krat hora estándar de krasnoyarsk +lint hora de las espóradas ecuatoriales +magt hora estándar de magadán +mart hora de las islas marquesas +mmt hora de birmania +msk hora estándar de moscú +mut hora estándar de mauricio +mvt hora de maldivas +myt hora de malasia +nct hora estándar de nueva caledonia +ndt hora de verano de terranova +nft hora de las islas norfolk +vot hora estándar de novosibirsk +npt hora de nepal +nrt hora de nauru +nst hora estándar de terranova +nzdt hora de verano de nueva zelanda +hzst hora estándar de nueva zelanda +omst hora estándar de omsk +orat hora de kazajistán occidental +pett hora de kamchatka +pgt hora de papúa nueva guinea +phot hora de las islas fénix +pkt hora estándar de pakistán +pmdt hora de verano de san pedro y miquelón +pmst hora estándar de san predo y miquelón +pont hora de pohnpei +pwt hora de palaos +qyzt hora de quizilordá +ret hora de reunión +sakt hora estándar de sajalín +samt hora de samara +sast hora de sudáfrica +sbt hora de las islas salomón +sret hora de srednekolimsk +srt hora de surinam +sst hora estándar de samoa +taht hora de tahití +tft hora de las tierras australes y antárticas francesas +tjt hora de tayikistán +tkt hora de tokelau +tlt hora de timor oriental +tmt hora estándar de turkmenistán +tot hora estándar de tonga +trt hora estándar de turquía +tvt hora de tuvalu +ulat hora estándar de ulán bator +uzt hora estándar de uzbekistán +vlat hora estándar de vladivostok +volt hora estándar de volgogrado +vost hora de vostok +vut hora estándar de vanuatu +wakt hora de la isla wake +wat hora estándar de áfrica occidental +wgst hora de verano de groenlandia occidental +wgt hora estándar de groenlandia occidental +wib hora de indonesia occidental +wit hora de indonesia oriental +wita hora de indonesia central +wst hora de samoa occidental +yakt hora estándar de yakutsk +yekt hora estándar de ekaterimburgo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f86e6d1a5d78f40fec101c35e04fcbfd3337256b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv @@ -0,0 +1,62 @@ +ud. usted +Ud. usted +uds. ustedes +Uds. ustedes +Vd. vosotros +Vds. vosotros +vd. vosotros +vds. vosotros +D. Don +Da. Doña +Dr. Doctor +Dra. Doctora +dr. doctor +dra. doctora +d. don +da. doña +EE. UU. Estados Unidos +ee. uu. estados unidos +Gob. gobierno +gob. gobierno +esq. esquina +Av. avenida +Avda. avenida +av. avenida +avda. avenida +avd. avenida +pág. página +p.ej. por ejemplo +p. ej. por ejemplo +Prof. profesor +Profa. profesora +prof. profesor +profa. profesora +S sur +N norte +Sr. señor +sr. señor +Sra. señora +sra. señora +Srta. señorita +srta. señorita +tel. teléfono +arr.º arrollo +blvr. bulevar +blvd. bulevar +c/ calle +c.° camino +c.º a. camino alto +c.º b. camino bajo +c.º v. camino viejo +carr.ª carrera +ctra. carretera +carret. carretera +cta. cuenta +pje. pasaje +p.º paseo +p.º a. paseo alto +p.º b. paseo bajo +pza. plaza +plza. plaza +rbla. rambla +vda. vereda diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ec539d1833f3c93097edd098d0570db6416ac3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/graph_utils.py @@ -0,0 +1,218 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, NEMO_SPACE +from zonos2.vendor.nemo_text_processing.text_normalization.es import LOCALIZATION +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input") +tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input") +teens = pynini.project(pynini.string_file(get_abs_path("data/numbers/teen.tsv")), "input") +twenties = pynini.project(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")), "input") +hundreds = pynini.project(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")), "input") + +accents = pynini.string_map([("á", "a"), ("é", "e"), ("í", "i"), ("ó", "o"), ("ú", "u")]) + +if LOCALIZATION == "am": # Setting localization for central and northern america formatting + cardinal_separator = pynini.string_map([",", NEMO_SPACE]) + decimal_separator = pynini.accep(".") +else: + cardinal_separator = pynini.string_map([".", NEMO_SPACE]) + decimal_separator = pynini.accep(",") + +ones = pynini.union("un", "ún") +fem_ones = pynini.union(pynini.cross("un", "una"), pynini.cross("ún", "una"), pynini.cross("uno", "una")) +one_to_one_hundred = pynini.union(digits, "uno", tens, teens, twenties, tens + pynini.accep(" y ") + digits) +fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", NEMO_SIGMA) + + +ES_MINUS = pynini.union("menos", "Menos", "MENOS").optimize() +ES_PLUS = pynini.union("más", "Más", "MÁS").optimize() + + +def strip_accent(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Converts all accented vowels to non-accented equivalents + + Args: + fst: Any fst. Composes vowel conversion onto fst's output strings + """ + return fst @ pynini.cdrewrite(accents, "", "", NEMO_SIGMA) + + +def shift_cardinal_gender(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and + Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies + to value place for <1000 and multiple of 1000. (e.g. "doscientos mil doscientos" -> "doscientas mil doscientas".) For place values greater than the thousands, there + is no gender shift as the higher powers of ten ("millones", "billones") are masculine nouns and any conversion would be formally + ungrammatical. + e.g. + "doscientos" -> "doscientas" + "doscientos mil" -> "doscientas mil" + "doscientos millones" -> "doscientos millones" + "doscientos mil millones" -> "doscientos mil millones" + "doscientos millones doscientos mil doscientos" -> "doscientos millones doscientas mil doscientas" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + before_mil = ( + NEMO_SPACE + + (pynini.accep("mil") | pynini.accep("milésimo")) + + pynini.closure(NEMO_SPACE + hundreds, 0, 1) + + pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) + + pynini.union(pynini.accep("[EOS]"), pynini.accep('"'), decimal_separator) + ) + before_double_digits = pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union( + pynini.accep("[EOS]"), pynini.accep('"') + ) + + fem_allign = pynini.cdrewrite(fem_hundreds, "", before_mil, NEMO_SIGMA) # doscientas mil dosciento + fem_allign @= pynini.cdrewrite(fem_hundreds, "", before_double_digits, NEMO_SIGMA) # doscientas mil doscienta + + fem_allign @= pynini.cdrewrite( + fem_ones, "", pynini.union("[EOS]", '"', decimal_separator), NEMO_SIGMA + ) # If before a quote or EOS, we know it's the end of a string + + return fst @ fem_allign + + +def shift_number_gender(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to + feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una". + This has no boundary restriction and will perform shift across all values in output string. + e.g. + "doscientos" -> "doscientas" + "doscientos millones" -> "doscientas millones" + "doscientos millones doscientos" -> "doscientas millones doscientas" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + fem_allign = pynini.cdrewrite(fem_hundreds, "", "", NEMO_SIGMA) + fem_allign @= pynini.cdrewrite( + fem_ones, + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) # If before a quote or EOS, we know it's the end of a string + + return fst @ fem_allign + + +def strip_cardinal_apocope(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only + affects strings where the final value is a variation of "un". + e.g. + "un" -> "uno" + "veintiún" -> "veintiuno" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + # Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one + strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno") + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) + return fst @ strip + + +def add_cardinal_apocope_fem(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable + with text string, so is included for non-deterministic cases. + e.g. + "una" -> "un" + "veintiuna" -> "veintiun" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + # Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one + strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún") + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) + return fst @ strip + + +def roman_to_int(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. + e.g. + "V" -> "5" + "i" -> "1" + + Args: + fst: Any fst. Composes fst onto Roman conversion outputs. + """ + + def _load_roman(file: str, upper_casing: bool): + roman = load_labels(get_abs_path(file)) + if upper_casing: + roman_numerals = [(x.upper(), y) for x, y in roman] + else: + roman_numerals = [(x, y) for x, y in roman] + return pynini.string_map(roman_numerals) + + # A split between all upper-case and all lower-case Roman numerals is introduced in order to preserve orthographic accuracy, + # and to prevent cases in which certain proper nouns e.g. (Li, Xi, Yi, etc.) are transduced to Roman numerals. + + digit_lower = _load_roman("data/roman/digit.tsv", False) + digit_upper = _load_roman("data/roman/digit.tsv", True) + ties_lower = _load_roman("data/roman/ties.tsv", False) + ties_upper = _load_roman("data/roman/ties.tsv", True) + hundreds_lower = _load_roman("data/roman/hundreds.tsv", False) + hundreds_upper = _load_roman("data/roman/hundreds.tsv", True) + thousands_lower = _load_roman("data/roman/thousands.tsv", False) + thousands_upper = _load_roman("data/roman/thousands.tsv", True) + + graph = ( + (digit_upper | digit_lower) + | ( + (ties_upper + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01))) + | (ties_lower + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01))) + ) + | ( + ( + hundreds_upper + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + hundreds_lower + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ) + | ( + ( + thousands_upper + + (hundreds_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + thousands_lower + + (hundreds_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ) + ).optimize() + + return graph @ fst diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..6b3818ccf0d944916a047b7767adc879e74205fe --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py @@ -0,0 +1,177 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) +ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))) +twenties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/twenties.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils) + and converts to a string of digits: + "1 000" -> "1000" + "1.000.000" -> "1000000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "mil" } + "2.000.000" -> cardinal { integer: "dos millones" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Any single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + + # Any double digit + graph_tens = teen + graph_tens |= ties + (pynutil.delete('0') | (pynutil.insert(" y ") + graph_digit)) + graph_tens |= twenties + + self.tens = graph_tens.optimize() + + self.two_digit_non_zero = pynini.union( + graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit) + ).optimize() + + # Three digit strings + graph_hundreds = hundreds + pynini.union( + pynutil.delete("00"), (insert_space + graph_tens), (pynini.cross("0", NEMO_SPACE) + graph_digit) + ) + graph_hundreds |= pynini.cross("100", "cien") + graph_hundreds |= ( + pynini.cross("1", "ciento") + insert_space + pynini.union(graph_tens, pynutil.delete("0") + graph_digit) + ) + + self.hundreds = graph_hundreds.optimize() + + # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit + ) + graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + + graph_thousands_component_at_least_one_none_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert(" mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert(" mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_million = pynutil.add_weight(pynini.cross("000001", "un millón"), -0.001) + graph_million |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" millones") + graph_million |= pynutil.delete("000000") + graph_million += insert_space + + graph_billion = pynutil.add_weight(pynini.cross("000001", "un billón"), -0.001) + graph_billion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" billones") + graph_billion |= pynutil.delete("000000") + graph_billion += insert_space + + graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trillón"), -0.001) + graph_trillion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" trillones") + graph_trillion |= pynutil.delete("000000") + graph_trillion += insert_space + + graph = ( + graph_trillion + + graph_billion + + graph_million + + (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000")) + ) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + ) + self.graph |= zero + + self.graph = filter_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..26a9775c129d0d5e57ce6952232c3f0a78271790 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/date.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_extra_space +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_cardinal_apocope +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +articles = pynini.union("de", "del", "el", "del año", "año") +delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT +month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv")) +month_abbr = pynini.string_file(get_abs_path("data/dates/months_abbr.tsv")) +weekdays = pynini.string_file(get_abs_path("data/dates/days_abbr.tsv")) +year_suffix = pynini.string_file(get_abs_path("data/dates/year_suffix.tsv")) + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "01.04.2010" -> date { day: "un" month: "enero" year: "dos mil diez" preserve_order: true } + "marzo 4 2000" -> date { month: "marzo" day: "cuatro" year: "dos mil" } + "1990-20-01" -> date { year: "mil novecientos noventa" day: "veinte" month: "enero" } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + number_to_month = month_numbers.optimize() + month_graph = pynini.project(number_to_month, "output") + + abbr_to_month = month_abbr.optimize() + month_graph |= abbr_to_month + + numbers = cardinal.graph + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + + # 01, 31, 1 + digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers + + # weekdays + optional_weekdays = pynini.closure(weekdays + NEMO_SPACE, 0, 1) + day = (pynutil.insert("day: \"") + optional_weekdays + digit_day + pynutil.insert("\"")).optimize() + + digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = digit_month @ number_to_month + + month_name = (pynutil.insert("month: \"") + month_graph + pynutil.insert("\"")).optimize() + month_number = (pynutil.insert("month: \"") + number_to_month + pynutil.insert("\"")).optimize() + + # prefer cardinal over year + year = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) # 90, 990, 1990 + year = strip_cardinal_apocope(year @ numbers) + self.year = year + + # handle roman centuries and years + roman_numbers = roman_to_int(numbers) + roman_centuries = pynini.union("siglo ", "año ") + roman_numbers + roman_centuries_graph = ( + pynutil.insert("year: \"") + + roman_centuries + + pynutil.insert("\"") + + pynutil.insert(" preserve_order: true") + ).optimize() + + year_only = pynutil.insert("year: \"") + year + pynutil.insert("\"") + year_with_articles = ( + pynutil.insert("year: \"") + pynini.closure(articles + NEMO_SPACE, 0, 1) + year + pynutil.insert("\"") + ) + + # handle years with eras + year_suffix_graph = ( + pynutil.insert("year: \"") + + (numbers | roman_centuries) + + pynini.closure(NEMO_SPACE, 0, 1) + + year_suffix + + pynutil.insert("\"") + ) + + graph_dmy = ( + day + + pynini.closure(pynutil.delete(" de")) + + NEMO_SPACE + + month_name + + pynini.closure(NEMO_SPACE + year_with_articles, 0, 1) + ) + + graph_mdy = ( # English influences on language + month_name + delete_extra_space + day + pynini.closure(NEMO_SPACE + year_with_articles, 0, 1) + ) + + separators = [".", "-", "/"] + for sep in separators: + year_optional = pynini.closure(pynini.cross(sep, NEMO_SPACE) + year_only, 0, 1) + new_graph = day + pynini.cross(sep, NEMO_SPACE) + month_number + year_optional + graph_dmy |= new_graph + if not deterministic: + new_graph = month_number + pynini.cross(sep, NEMO_SPACE) + day + year_optional + graph_mdy |= new_graph + + dash = "-" + day_optional = pynini.closure(pynini.cross(dash, NEMO_SPACE) + day, 0, 1) + graph_ymd = NEMO_DIGIT**4 @ year_only + pynini.cross(dash, NEMO_SPACE) + month_number + day_optional + + final_graph = graph_dmy + pynutil.insert(" preserve_order: true") + final_graph |= graph_ymd + final_graph |= graph_mdy + final_graph |= roman_centuries_graph + final_graph |= year_suffix_graph + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..df198ffb39b166c1f3fe73b627d35cb06ce31a76 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py @@ -0,0 +1,133 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + cardinal_separator, + decimal_separator, + strip_cardinal_apocope, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 2 millones -> integer_part: "dos" quantity: "millones" + e.g. 2,4 millones -> integer_part: "dos" fractional_part: "cuatro" quantity: "millones" + + Args: + decimal_graph: DecimalFST + cardinal_graph: CardinalFST + """ + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers # The cardinal we're passing only produces 'un' for one, so gender agreement is safe (all quantities are masculine). Limit to 10^6 power. + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -11,4006 billones -> decimal { negative: "true" integer_part: "once" fractional_part: "cuatro cero cero seis" quantity: "billones" preserve_order: true } + 1 billón -> decimal { integer_part: "un" quantity: "billón" preserve_order: true } + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.hundreds, cardinal.tens) + graph += pynini.closure(insert_space + graph) + + else: + # General pattern is 1-3 digits: map as cardinal, default to tens followed by digits otherwise \ + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.tens + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.tens + + pynini.closure(insert_space + cardinal.tens, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,010" + ) + + # Need to strip apocope everywhere BUT end of string + reverse_apocope = pynini.string_map([("un", "uno"), ("ún", "uno")]) + apply_reverse_apocope = pynini.cdrewrite(reverse_apocope, "", NEMO_SPACE, NEMO_SIGMA) + graph @= apply_reverse_apocope + + # Technically decimals should be space delineated groups of three, e.g. (1,333 333). This removes any possible spaces + strip_formatting = pynini.cdrewrite(delete_space, "", "", NEMO_SIGMA) + graph = strip_formatting @ graph + + self.graph = graph.optimize() + + graph_separator = pynutil.delete(decimal_separator) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph_fractional = pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"') + + # Integer graph maintains apocope except for ones place + graph_integer = ( + strip_cardinal_apocope(cardinal.graph) + if deterministic + else pynini.union(cardinal.graph, strip_cardinal_apocope(cardinal.graph)) + ) # Gives us forms w/ and w/o apocope + self.graph_integer = pynutil.insert('integer_part: "') + graph_integer + pynutil.insert('"') + final_graph_wo_sign = self.graph_integer + graph_separator + insert_space + self.graph_fractional + + self.final_graph_wo_negative = ( + final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal.graph).optimize() + ) + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..95e24723c6f2d6b7e9f0f6d07a051d77f4e6963f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py @@ -0,0 +1,107 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + protocol_string, + username_string, + www, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + full_stop_accep = pynini.accep(".") + full_stop = "." + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_full_stop = pynini.difference(symbols, full_stop_accep) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_full_stop), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + # domains + domain = full_stop_accep + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) + + # email + username = ( + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) + ) + email = username + domain_graph + + # social media tags + tag = ( + pynini.cross(at, "") + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) + ) + + # url + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) + # protocol_end = pynini.accep("www.") + protocol_end = ( + pynini.accep(www + full_stop) + if deterministic + else pynini.accep(www + full_stop) | pynini.cross(www + full_stop, "doble ve doble ve doble ve.") + ) + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..5d632691f3c38ef37a87a5773b50a355b6adb4b0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py @@ -0,0 +1,164 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +ordinal_exceptions = pynini.string_file(get_abs_path("data/fractions/ordinal_exceptions.tsv")) +higher_powers_of_ten = pynini.string_file(get_abs_path("data/fractions/powers_of_ten.tsv")) +fraction_symbols = pynini.string_file(get_abs_path("data/fractions/fraction_symbols.tsv")) + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "veintitrés" numerator: "cuatro" denominator: "quinto" mophosyntactic_features: "ordinal" } } + + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + ordinal_graph = ordinal.graph + + # 2-10 are all ordinals + three_to_ten = pynini.string_map( + [ + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + ] + ) + block_three_to_ten = pynutil.delete(three_to_ten) # To block cardinal productions + if not deterministic: # Multiples of tens are sometimes rendered as ordinals + three_to_ten |= pynini.string_map( + [ + "20", + "30", + "40", + "50", + "60", + "70", + "80", + "90", + ] + ) + graph_three_to_ten = three_to_ten @ ordinal_graph + graph_three_to_ten @= pynini.cdrewrite(ordinal_exceptions, "", "", NEMO_SIGMA) + + # Higher powers of tens (and multiples) are converted to ordinals. + hundreds = pynini.string_map( + [ + "100", + "200", + "300", + "400", + "500", + "600", + "700", + "800", + "900", + ] + ) + graph_hundreds = hundreds @ ordinal_graph + + multiples_of_thousand = ordinal.multiples_of_thousand # So we can have X milésimos + + graph_higher_powers_of_ten = ( + pynini.closure(ordinal.one_to_one_thousand + NEMO_SPACE, 0, 1) + + pynini.closure("mil ", 0, 1) + + pynini.closure(ordinal.one_to_one_thousand + NEMO_SPACE, 0, 1) + ) # x millones / x mil millones / x mil z millones + graph_higher_powers_of_ten += higher_powers_of_ten + graph_higher_powers_of_ten = cardinal_graph @ graph_higher_powers_of_ten + graph_higher_powers_of_ten @= pynini.cdrewrite( + pynutil.delete("un "), + pynini.accep("[BOS]"), + pynini.project(higher_powers_of_ten, "output"), + NEMO_SIGMA, + ) # we drop 'un' from these ordinals (millionths, not one-millionths) + + graph_higher_powers_of_ten = multiples_of_thousand | graph_hundreds | graph_higher_powers_of_ten + block_higher_powers_of_ten = pynutil.delete( + pynini.project(graph_higher_powers_of_ten, "input") + ) # For cardinal graph + + graph_fractions_ordinals = graph_higher_powers_of_ten | graph_three_to_ten + graph_fractions_ordinals += pynutil.insert( + '" morphosyntactic_features: "ordinal"' + ) # We note the root for processing later + + # Blocking the digits and hundreds from Cardinal graph + graph_fractions_cardinals = pynini.cdrewrite( + block_three_to_ten | block_higher_powers_of_ten, + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, + ) + graph_fractions_cardinals @= NEMO_CHAR.plus @ pynini.cdrewrite( + pynutil.delete("0"), + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, + ) # Empty characters become '0' for NEMO_CHAR fst, so need to block + graph_fractions_cardinals @= cardinal_graph + graph_fractions_cardinals += pynutil.insert( + '" morphosyntactic_features: "add_root"' + ) # blocking these entries to reduce erroneous possibilities in debugging + + if deterministic: + graph_fractions_cardinals = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ graph_fractions_cardinals + ) # Past hundreds the conventional scheme can be hard to read. For determinism we stop here + + graph_denominator = pynini.union( + graph_fractions_ordinals, + graph_fractions_cardinals, + pynutil.add_weight(cardinal_graph + pynutil.insert('"'), 0.001), + ) # Last form is simply recording the cardinal. Weighting so last resort + + integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + NEMO_SPACE + numerator = ( + pynutil.insert('numerator: "') + cardinal_graph + (pynini.cross("/", '" ') | pynini.cross(" / ", '" ')) + ) + denominator = pynutil.insert('denominator: "') + graph_denominator + + fraction = (fraction_symbols @ (numerator + denominator)) | (numerator + denominator) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph = optional_minus_graph + pynini.closure(integer, 0, 1) + fraction + + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..ed9ef5e4248d366cd264b31830715f457488ebcb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py @@ -0,0 +1,216 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +unit = pynini.string_file(get_abs_path("data/measures/measurements.tsv")) +unit_complex = pynini.string_file(get_abs_path("data/measures/measurements_complex.tsv")) +unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv")) +unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv")) +math_symbols = pynini.string_file(get_abs_path("data/measures/math_symbols.tsv")) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "2,4 g" -> measure { cardinal { integer_part: "dos" fractional_part: "cuatro" units: "gramos" preserve_order: true } } + "1 g" -> measure { cardinal { integer: "un" units: "gramo" preserve_order: true } } + "1 millón g" -> measure { cardinal { integer: "un quantity: "millón" units: "gramos" preserve_order: true } } + This class also converts words containing numbers and letters + e.g. "a-8" —> "a ocho" + e.g. "1,2-a" —> "uno coma dos a" + + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + unit_singular = unit + unit_plural = unit_singular @ (unit_plural_fem | unit_plural_masc) + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(unit_plural) + + optional_graph_negative = pynini.closure("-", 0, 1) + + unit_plural = pynutil.insert('units: "') + graph_unit_plural + pynutil.insert('"') + + unit_singular_graph = pynutil.insert('units: "') + graph_unit_singular + pynutil.insert('"') + + unit_complex_plural = unit_complex @ (unit_plural_fem | unit_plural_masc) + + graph_unit_complex = convert_space(unit_complex) + graph_unit_complex_plural = convert_space(unit_complex_plural) + + graph_unit_denominator = ( + pynini.cross("/", "por") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, + ) + + complex_unit_singular_graph = ( + pynutil.insert('units: "') + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator | graph_unit_complex) + + pynutil.insert('"') + ) + + complex_unit_plural_graph = ( + pynutil.insert('units: "') + + ((graph_unit_plural + optional_unit_denominator) | graph_unit_denominator | graph_unit_complex_plural) + + pynutil.insert('"') + ) + + subgraph_decimal = ( + decimal.fst + insert_space + pynini.closure(NEMO_SPACE, 0, 1) + (unit_plural | complex_unit_plural_graph) + ) + subgraph_cardinal = ( + (optional_graph_negative + (NEMO_SIGMA - "1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_plural | complex_unit_plural_graph) + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_singular_graph | complex_unit_singular_graph) + ) + + subgraph_fraction = ( + fraction.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_plural | complex_unit_singular_graph) + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(' } units: "') + + pynini.union("x", "X") + + pynutil.insert('"') + ) + + cardinal_times = ( + pynutil.insert('cardinal { integer: "') + + strip_cardinal_apocope(cardinal_graph) + + pynutil.insert('" } units: "') + + pynini.union("x", "X") + + pynutil.insert('"') + ) + + cardinal_dash_alpha = ( + pynutil.insert('cardinal { integer: "') + + strip_cardinal_apocope(cardinal_graph) + + pynutil.delete("-") + + pynutil.insert('" } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert('"') + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.delete("-") + + pynutil.insert(' } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete("-") + + pynutil.insert('"') + + pynutil.insert(' cardinal { integer: "') + + cardinal_graph + + pynutil.insert('" } preserve_order: true') + ) + + alpha_dash_decimal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete("-") + + pynutil.insert('"') + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + delimiter = pynini.accep(" ") | pynutil.insert(" ") + math_numbers = cardinal_graph | pynutil.add_weight(cardinal_graph @ pynini.cross("un", "uno"), -1) | NEMO_ALPHA + math = ( + math_numbers + + pynini.closure(delimiter + math_symbols + delimiter + math_numbers, 1) + + delimiter + + pynini.cross("=", "es igual a") + + delimiter + + math_numbers + ) + + math |= ( + math_numbers + + delimiter + + pynini.cross("=", "es igual a") + + math_numbers + + pynini.closure(delimiter + math_symbols + delimiter + math_numbers, 1) + ) + + math = ( + pynutil.insert("units: \"math\" cardinal { integer: \"") + + math + + pynutil.insert("\" } preserve_order: true") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | subgraph_fraction + | decimal_times + | cardinal_times + | alpha_dash_decimal + | math + ) + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..635bc24b437be7652369be74b17f4de0cb0738b3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/money.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import decimal_separator +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) +fem_plural = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc_plural = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "€1" -> money { currency_maj: "euro" integer_part: "un"} + "€1,000" -> money { currency_maj: "euro" integer_part: "un" } + "€1,001" -> money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un" } + "£1,4" -> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true } + -> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "penique" preserve_order: true } + "0,01 £" -> money { fractional_part: "un" currency_min: "penique" preserve_order: true } + "0,02 £" -> money { fractional_part: "dos" currency_min: "peniques" preserve_order: true } + "£0,01 million" -> money { currency_maj: "libra" integer_part: "cero" fractional_part: "cero un" quantity: "million" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + maj_singular_graph = maj_singular + min_singular_graph = min_singular + maj_plural_graph = maj_singular @ (fem_plural | masc_plural) + min_plural_graph = min_singular @ (fem_plural | masc_plural) + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_singular_graph + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_plural_graph + pynutil.insert("\"") + + graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "un") + pynutil.insert("\"") + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, # 1,05 $ + ) + graph_decimal_plural = ( + (NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA + ) @ graph_decimal_plural # Can't have "un euros" + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, # 1,05 $ + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") + ) + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert("currency_min: \"") + min_singular_graph + pynutil.insert("\"") + graph_min_plural = pynutil.insert("currency_min: \"") + min_plural_graph + pynutil.insert("\"") + + # format ** euro ** cent + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + # non zero integer part + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ pynini.cross("1", "un") + + pynutil.insert("\"") + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_non_zero + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.two_digit_non_zero + + pynutil.insert("\""), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..7344c62935ccf3ede889032380da2e678e090292 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py @@ -0,0 +1,172 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_accent +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +digit = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/digit.tsv"))) +teens = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/teen.tsv"))) +twenties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/twenties.tsv"))) +ties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/ties.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/hundreds.tsv"))) +gender_suffix = pynini.string_file(get_abs_path("data/ordinals/gender_suffix.tsv")) + + +def get_one_to_one_thousand(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Produces an acceptor for verbalizations of all numbers from 1 to 1000. Needed for ordinals and fractions. + + Args: + cardinal: CardinalFst + + Returns: + fst: A pynini.FstLike object + """ + numbers = pynini.string_map([str(_) for _ in range(1, 1000)]) @ cardinal + return pynini.project(numbers, "output").optimize() + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + "21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "gender_masc" } + This class converts ordinal up to the millionth (millonésimo) order (exclusive). + + This FST also records the ending of the ordinal (called "morphosyntactic_features"): + either as gender_masc, gender_fem, or apocope. Also introduces plural feature for non-deterministic graphs. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify") + cardinal_graph = cardinal.graph + + graph_digit = digit.optimize() + graph_teens = teens.optimize() + graph_ties = ties.optimize() + graph_twenties = twenties.optimize() + graph_hundreds = hundreds.optimize() + + if not deterministic: + # Some alternative derivations + graph_ties = graph_ties | pynini.cross("sesenta", "setuagésimo") + + graph_teens = graph_teens | pynini.cross("once", "decimoprimero") + graph_teens |= pynini.cross("doce", "decimosegundo") + + graph_digit = graph_digit | pynini.cross("nueve", "nono") + graph_digit |= pynini.cross("siete", "sétimo") + + graph_tens_component = ( + graph_teens + | (graph_ties + pynini.closure(pynini.cross(" y ", NEMO_SPACE) + graph_digit, 0, 1)) + | graph_twenties + ) + + graph_hundred_component = pynini.union( + graph_hundreds + pynini.closure(NEMO_SPACE + pynini.union(graph_tens_component, graph_digit), 0, 1), + graph_tens_component, + graph_digit, + ) + + # Need to go up to thousands for fractions + self.one_to_one_thousand = get_one_to_one_thousand(cardinal_graph) + + thousands = pynini.cross("mil", "milésimo") + + graph_thousands = ( + strip_accent(self.one_to_one_thousand) + NEMO_SPACE + thousands + ) # Cardinals become prefix for thousands series. Snce accent on the powers of ten we strip accent from leading words + graph_thousands @= pynini.cdrewrite(delete_space, "", "milésimo", NEMO_SIGMA) # merge as a prefix + graph_thousands |= thousands + + self.multiples_of_thousand = (cardinal_graph @ graph_thousands).optimize() + + if ( + not deterministic + ): # Formally the words preceding the power of ten should be a prefix, but some maintain word boundaries. + graph_thousands |= (self.one_to_one_thousand @ graph_hundred_component) + NEMO_SPACE + thousands + + graph_thousands += pynini.closure(NEMO_SPACE + graph_hundred_component, 0, 1) + + ordinal_graph = graph_thousands | graph_hundred_component + ordinal_graph = cardinal_graph @ ordinal_graph + + if not deterministic: + # The 10's and 20's series can also be two words + split_words = pynini.cross("decimo", "décimo ") | pynini.cross("vigesimo", "vigésimo ") + split_words = pynini.cdrewrite(split_words, "", NEMO_CHAR, NEMO_SIGMA) + ordinal_graph |= ordinal_graph @ split_words + + # If "octavo" is preceeded by the "o" within string, it needs deletion + ordinal_graph @= pynini.cdrewrite(pynutil.delete("o"), "", "octavo", NEMO_SIGMA) + + self.graph = ordinal_graph.optimize() + + delete_period = pynini.closure(pynutil.delete("."), 0, 1) # Sometimes the period is omitted + convert_gender_suffix = delete_period + gender_suffix + + # Managing Romanization, excluding words that may be ambiguous + roman_ordinals = roman_to_int(ordinal_graph) + # exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi") + exceptions = pynini.string_file(get_abs_path("data/ordinals/roman_exceptions.tsv")) + graph_exception = pynini.project(exceptions, 'input') + roman_ordinals = (pynini.project(roman_ordinals, "input") - graph_exception.arcsort()) @ roman_ordinals + + graph_roman = pynutil.insert("integer: \"") + roman_ordinals + pynutil.insert("\"") + + if not deterministic: + # Introduce plural + plural = pynini.closure(pynutil.insert("/plural"), 0, 1) + convert_gender_suffix += plural + + # Romanizations have no morphology marker, so in non-deterministic case we provide option for all + insert_morphology = pynutil.insert(convert_gender_suffix) + plural + insert_morphology = ( + pynutil.insert(" morphosyntactic_features: \"") + insert_morphology + pynutil.insert("\"") + ) + + graph_roman += insert_morphology + + else: + # We insert both genders as default + graph_roman += pynutil.insert(" morphosyntactic_features: \"gender_masc\"") | pynutil.insert( + " morphosyntactic_features: \"gender_fem\"" + ) + + graph = ( + pynutil.insert("integer: \"") + + ordinal_graph + + pynutil.insert("\"") + + pynutil.insert(" morphosyntactic_features: \"") + + convert_gender_suffix + + pynutil.insert("\"") + ) + graph = pynini.union(graph, graph_roman) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..06920d25c2309091f4438efd80a16600c3e43480 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py @@ -0,0 +1,194 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ones +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) +graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) +graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + 123-123-5678 -> { number_part: "uno dos tres uno dos tres cinco seis siete ocho" }. + In Spanish, digits are generally read individually, or as 2-digit numbers, + eg. "123" = "uno dos tres", + "1234" = "doce treinta y cuatro". + This will verbalize sequences of 10 (3+3+4 e.g. 123-456-7890). + 9 (3+3+3 e.g. 123-456-789) and 8 (4+4 e.g. 1234-5678) digits. + + (we ignore more complicated cases such as "doscientos y dos" or "tres nueves"). + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify") + + # create `single_digits` and `double_digits` graphs as these will be + # the building blocks of possible telephone numbers + single_digits = pynini.invert(graph_digit).optimize() | pynini.cross("0", "cero") + single_digits @= pynini.cdrewrite(pynini.cross(ones, "uno"), "", "", NEMO_SIGMA) + + # Any double digit + teen = pynini.invert(graph_teen) + ties = pynini.invert(graph_ties) + twenties = pynini.invert(graph_twenties) + + double_digits = teen + double_digits |= ties + (pynutil.delete('0') | (pynutil.insert(" y ") + single_digits)) + double_digits |= twenties + + # define separators + separators = pynini.union("-", " ", ".") + delete_separator = pynini.closure(pynutil.delete(separators), 0, 1) + + # process country codes as '+1' -> country_code: "one" + triple_number = pynini.closure(single_digits + insert_space, 2, 2) + single_digits + country_code = pynini.closure(pynini.cross("+", "más "), 0, 1) + ( + single_digits | double_digits | triple_number + ) + + # add ip and telephone prompts to this tag (as is in EN) + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + tel_prompt_sequence = telephone_prompts + NEMO_SPACE + pynini.closure(country_code, 0, 1) + + country_code_graph = ( + pynutil.insert("country_code: \"") + + (country_code | ip_prompts | tel_prompt_sequence) + + delete_separator + + pynutil.insert("\"") + ) + + # process IP addresses + digit_to_str_graph = single_digits + pynini.closure(pynutil.insert(" ") + single_digits, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " punto ") + digit_to_str_graph) ** 3 + + # process area codes with or without parentheses i.e. "212" or (212) + area_code = ( + pynini.closure(pynutil.delete("("), 0, 1) + + pynini.closure(single_digits + insert_space, 3, 3) + + pynini.closure(pynutil.delete(")"), 0, 1) + ) + + # process extensions + delete_ext = pynini.closure(pynutil.delete("ext."), 0, 1) + ext_graph = ( + pynutil.insert("extension: \"") + + delete_separator + + delete_ext + + delete_separator + + pynutil.insert("extensión ") + + pynini.closure(single_digits + insert_space, 1, 3) + + single_digits + + pynutil.insert("\"") + ) + + # define `ten_digit_graph`, `nine_digit_graph`, `eight_digit_graph` + # which produces telephone numbers spoken (1) only with single digits, + # or (2) spoken with double digits (and sometimes single digits) + + # 10-digit option (1): all single digits + ten_digit_graph = ( + area_code + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + single_digits + ) + + # 9-digit option (1): all single digits + nine_digit_graph = ( + area_code + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + delete_separator + + pynini.closure(single_digits + insert_space, 2, 2) + + single_digits + ) + + # 8-digit option (1): all single digits + eight_digit_graph = ( + pynini.closure(area_code, 0, 1) + + delete_separator + + pynini.closure(single_digits + insert_space, 4, 4) + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + single_digits + ) + + if not deterministic: + # 10-digit option (2): (1+2) + (1+2) + (2+2) digits + ten_digit_graph |= ( + pynini.closure(single_digits + insert_space + double_digits + insert_space + delete_separator, 2, 2) + + double_digits + + insert_space + + double_digits + ) + + # 9-digit option (2): (1+2) + (1+2) + (1+2) digits + nine_digit_graph |= ( + pynini.closure(single_digits + insert_space + double_digits + insert_space + delete_separator, 2, 2) + + single_digits + + insert_space + + double_digits + ) + + # 8-digit option (2): (2+2) + (2+2) digits + eight_digit_graph |= ( + double_digits + + insert_space + + double_digits + + insert_space + + delete_separator + + double_digits + + insert_space + + double_digits + ) + + # handle numbers with letters like "1-800-go-u-haul" + num_letter_area_code = area_code @ pynini.cross("ocho cero cero ", "ochocientos ") + number_word = pynini.closure(single_digits | NEMO_ALPHA, 1, 8) + number_words = pynini.closure(number_word + pynini.cross(separators, " "), 0, 2) + number_word + + nums_w_letters_graph = pynutil.add_weight(num_letter_area_code + delete_separator + number_words, 0.01) + + number_part = pynini.union( + ten_digit_graph, nine_digit_graph, eight_digit_graph, nums_w_letters_graph, ip_graph + ) + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + + graph = ( + pynini.closure(country_code_graph + delete_separator + insert_space, 0, 1) + + number_part + + pynini.closure(delete_separator + insert_space + ext_graph, 0, 1) + ) + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..0a5aa5f5201f2c875b2a57952624eb66ea925000 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/time.py @@ -0,0 +1,226 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +time_zones = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) +suffix = pynini.string_file(get_abs_path("data/time/time_suffix.tsv")) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + "02:15 est" -> time { hours: "dos" minutes: "quince" zone: "e s t"} + "2 h" -> time { hours: "dos" } + "9 h" -> time { hours: "nueve" } + "02:15:10 h" -> time { hours: "dos" minutes: "quince" seconds: "diez"} + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + delete_time_delimiter = pynutil.delete(pynini.union(".", ":")) + + fem_one = pynini.string_map([("un", "una"), ("ún", "una")]) + change_one = pynini.cdrewrite(fem_one, "", "", NEMO_SIGMA) + cardinal_graph_fem = cardinal.graph @ change_one + cardinal_graph_masc = cardinal.graph + + day_suffix = pynutil.insert("suffix: \"") + suffix + pynutil.insert("\"") + day_suffix = delete_space + insert_space + day_suffix + + delete_hora_suffix = delete_space + insert_space + pynutil.delete("h") + delete_minute_suffix = delete_space + insert_space + (pynutil.delete("min") | pynutil.delete("m")) + delete_second_suffix = delete_space + insert_space + (pynutil.delete("seg") | pynutil.delete("s")) + + labels_hour_24 = [ + str(x) for x in range(0, 25) + ] # Can see both systems. Twelve hour requires am/pm for ambiguity resolution + labels_hour_12 = [str(x) for x in range(1, 13)] + labels_minute_or_second = [str(x) for x in range(1, 60)] + + delete_leading_zero_to_double_digit = ( + pynini.closure(pynutil.delete("0") | (NEMO_DIGIT - "0"), 0, 1) + NEMO_DIGIT + ) + + graph_24 = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ delete_leading_zero_to_double_digit @ pynini.union(*labels_hour_24) + ) + graph_12 = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ delete_leading_zero_to_double_digit @ pynini.union(*labels_hour_12) + ) + graph_minute_or_second = ( + pynini.closure(NEMO_DIGIT, 1, 2) + @ delete_leading_zero_to_double_digit + @ pynini.union(*labels_minute_or_second) + ) + graph_hour_24 = graph_24 @ cardinal_graph_fem + graph_hour_12 = graph_12 @ cardinal_graph_fem + graph_minute_or_second @= cardinal_graph_masc + + final_graph_hour_only_24 = ( + pynutil.insert("hours: \"") + graph_hour_24 + pynutil.insert("\"") + delete_hora_suffix + ) + final_graph_hour_only_12 = pynutil.insert("hours: \"") + graph_hour_12 + pynutil.insert("\"") + day_suffix + + final_graph_hour_24 = pynutil.insert("hours: \"") + graph_hour_24 + pynutil.insert("\"") + final_graph_hour_12 = pynutil.insert("hours: \"") + graph_hour_12 + pynutil.insert("\"") + + final_graph_minute = pynutil.insert("minutes: \"") + graph_minute_or_second + pynutil.insert("\"") + final_graph_second = pynutil.insert("seconds: \"") + graph_minute_or_second + pynutil.insert("\"") + + # handle suffixes like gmt or utc+1 + utc_one = pynini.string_map([("un", "uno"), ("ún", "uno")]) + change_utc_one = pynini.cdrewrite(utc_one, "", "", NEMO_SIGMA) + utc_cardinal_graph = cardinal.graph @ change_utc_one + + utc_or_gmt_numbers = (NEMO_DIGIT @ utc_cardinal_graph) | ( + graph_hour_24 + delete_time_delimiter + pynutil.insert(" ") + graph_minute_or_second + ) + + utc_or_gmt_diff = ( + pynutil.delete(pynini.closure(" ", 0, 1)) + + pynutil.insert(" ") + + pynini.string_map([("+", "más"), ("-", "menos")]) + + pynutil.delete(pynini.closure(" ", 0, 1)) + + pynutil.insert(" ") + + utc_or_gmt_numbers + ) + + time_zone_graph = time_zones + pynini.closure(utc_or_gmt_diff, 0, 1) + + final_time_zone_optional = pynini.closure( + delete_space + insert_space + pynutil.insert("zone: \"") + time_zone_graph + pynutil.insert("\""), + 0, + 1, + ) + + # 02.30 h + graph_hm = ( + final_graph_hour_24 + + delete_time_delimiter + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + pynini.closure( + (delete_time_delimiter + (pynutil.delete("00") | (insert_space + final_graph_second))), 0, 1 + ) + + pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous + + final_time_zone_optional + ) + + # 2 h 30 min + graph_hm |= ( + final_graph_hour_24 + + delete_hora_suffix + + delete_space + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + delete_minute_suffix + + pynini.closure( + (delete_space + (pynutil.delete("00") | (insert_space + final_graph_second)) + delete_second_suffix), + 0, + 1, + ) + + final_time_zone_optional + ) + + # 2.30 a. m. (Only for 12 hour clock) + graph_hm |= ( + final_graph_hour_12 + + delete_time_delimiter + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + pynini.closure( + (delete_time_delimiter + (pynutil.delete("00") | (insert_space + final_graph_second))), 0, 1 + ) + + day_suffix + + final_time_zone_optional + ) + + graph_h = ( + pynini.union(final_graph_hour_only_24, final_graph_hour_only_12) + final_time_zone_optional + ) # Should always have a time indicator, else we'll pass to cardinals + + if not deterministic: + # This includes alternate vocalization (hour menos min, min para hour), here we shift the times and indicate a `style` tag + hour_shift_24 = pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to_24.tsv"))) + hour_shift_12 = pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to_12.tsv"))) + minute_shift = pynini.string_file(get_abs_path("data/time/minute_to.tsv")) + + graph_hour_to_24 = graph_24 @ hour_shift_24 @ cardinal_graph_fem + graph_hour_to_12 = graph_12 @ hour_shift_12 @ cardinal_graph_fem + + graph_minute_to = minute_shift @ graph_minute_or_second + + final_graph_hour_to_24 = pynutil.insert("hours: \"") + graph_hour_to_24 + pynutil.insert("\"") + final_graph_hour_to_12 = pynutil.insert("hours: \"") + graph_hour_to_12 + pynutil.insert("\"") + + final_graph_minute_to = pynutil.insert("minutes: \"") + graph_minute_to + pynutil.insert("\"") + + graph_menos = pynutil.insert(" style: \"1\"") + graph_para = pynutil.insert(" style: \"2\"") + + final_graph_style = graph_menos | graph_para + + # 02.30 h (omitting seconds since a bit awkward) + graph_hm |= ( + final_graph_hour_to_24 + + delete_time_delimiter + + insert_space + + final_graph_minute_to + + pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous + + final_time_zone_optional + + final_graph_style + ) + + # 2 h 30 min + graph_hm |= ( + final_graph_hour_to_24 + + delete_hora_suffix + + delete_space + + insert_space + + final_graph_minute_to + + delete_minute_suffix + + final_time_zone_optional + + final_graph_style + ) + + # 2.30 a. m. (Only for 12 hour clock) + graph_hm |= ( + final_graph_hour_to_12 + + delete_time_delimiter + + insert_space + + final_graph_minute_to + + day_suffix + + final_time_zone_optional + + final_graph_style + ) + + final_graph = graph_hm | graph_h + if deterministic: + final_graph = final_graph + pynutil.insert(" preserve_order: true") + final_graph = final_graph.optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..0c39fa7da6a17ed74bc88229438f297142cf53ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py @@ -0,0 +1,161 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.decimals import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State aRchive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_es_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic) + ordinal_graph = self.ordinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) + fraction_graph = self.fraction.fst + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, + ) + measure_graph = self.measure.fst + self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + word_graph = WordFst(deterministic=deterministic).fst + self.time = TimeFst(self.cardinal, deterministic=deterministic) + time_graph = self.time.fst + self.telephone = TelephoneFst(deterministic=deterministic) + telephone_graph = self.telephone.fst + self.electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = self.electronic.fst + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + money_graph = self.money.fst + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.09) + | pynutil.add_weight(measure_graph, 1.08) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.09) + | pynutil.add_weight(telephone_graph, 1.11) + | pynutil.add_weight(electronic_graph, 1.11) + | pynutil.add_weight(word_graph, 200) + ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct), + 1, + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..3c5da88b44178abe03852898518cef68c4a78606 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "sr." -> tokens { name: "señor" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measures/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..efe49c4b592ae29127ba5af1524a7a0d81b34055 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/taggers/word.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. dormir -> tokens { name: "dormir" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..375bc1fb1f3e7593371e8f56ffa1cb0c543ff5c2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..8afcaf4a3a3bcd28fc177d221e122a45eac3ed15 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + add_cardinal_apocope_fem, + shift_cardinal_gender, + strip_cardinal_apocope, +) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "dos" } -> "dos" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "menos "), 0, 1) + self.optional_sign = optional_sign + + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + + graph_masc = optional_sign + integer + graph_fem = shift_cardinal_gender(graph_masc) + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + # Adding adjustment for fem gender (choice of gender will be random) + graph = graph_masc | graph_fem + + if not deterministic: + # For alternate renderings when apocope is omitted (i.e. cardinal stands alone) + graph |= strip_cardinal_apocope(graph_masc) + # "una" will drop to "un" in unique contexts + graph |= add_cardinal_apocope_fem(graph_fem) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..3d351b1a2e31c65220137c06b3955e77c15cae50 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py @@ -0,0 +1,83 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope +from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.date import articles + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "treinta y uno" month: "marzo" year: "dos mil" } -> "treinta y uno de marzo de dos mil" + date { day: "uno" month: "mayo" year: "del mil novecientos noventa" } -> "primero de mayo del mil novecientos noventa" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day_cardinal = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + day = strip_cardinal_apocope(day_cardinal) + + primero = pynini.cdrewrite(pynini.cross("uno", "primero"), "[BOS]", "[EOS]", NEMO_SIGMA) + day = ( + (day @ primero) if deterministic else pynini.union(day, day @ primero) + ) # Primero for first day is traditional, but will vary depending on region + + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + year = ( + pynutil.delete("year: \"") + + articles + + NEMO_SPACE + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # Insert preposition if wasn't originally with the year. This would mean a space was present + year = pynutil.add_weight(year, -0.001) + year |= ( + pynutil.delete("year: \"") + + pynutil.insert("de ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # Roman centuries + graph_roman_centuries = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # day month year + graph_dmy = day + pynini.cross(NEMO_SPACE, " de ") + month + pynini.closure(pynini.accep(" ") + year, 0, 1) + + graph_mdy = month + NEMO_SPACE + day + pynini.closure(NEMO_SPACE + year, 0, 1) + if deterministic: + graph_mdy += pynutil.delete(" preserve_order: true") # Only accepts this if was explicitly passed + + self.graph = graph_dmy | graph_mdy | graph_roman_centuries + final_graph = self.graph + delete_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..da9cb55d61401d6bdb01abf7fde3b85abe0714e2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es import LOCALIZATION +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + add_cardinal_apocope_fem, + shift_cardinal_gender, + shift_number_gender, + strip_cardinal_apocope, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "menos ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" punto ") if LOCALIZATION == "am" else pynutil.insert(" coma ") + if not deterministic: + conjunction |= pynutil.insert(pynini.union(" con ", " y ")) + fractional_default |= strip_cardinal_apocope(fractional_default) + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph_masc = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + # Allowing permutation for fem gender, don't include quantity since "million","billion", etc.. are masculine + graph_fem = optional_sign + (shift_cardinal_gender(integer) + delete_space + shift_number_gender(fractional)) + if not deterministic: # "una" will drop to "un" in certain cases + graph_fem |= add_cardinal_apocope_fem(graph_fem) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + self.graph_masc = (graph_masc + delete_preserve_order).optimize() + self.graph_fem = (graph_fem + delete_preserve_order).optimize() + + graph = graph_masc | graph_fem + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..f50940572c759c62af0cd3514399026d58f70faf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + colon, + delete_preserve_order, + domain_string, + double_quotes, + protocol_string, + username_string, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +arroba = "arroba" + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c arroba hotmail punto com" + -> "a b c arroba h o t m a i l punto c o m" + -> "a b c arroba hotmail punto c o m" + -> "a b c at h o t m a i l punto com" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit_no_zero = ( + digit_no_zero @ pynini.cdrewrite(pynini.cross("un", "uno"), "", "", NEMO_SIGMA).optimize() + ) + graph_digit = graph_digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) + user_name @= verbalize_characters + + convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) + protocol = ( + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete(double_quotes) + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert(arroba + NEMO_SPACE) + domain + | (pynutil.insert(arroba + NEMO_SPACE) + user_name) + ) + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..a1a52bc143253b6916426935e096b1358ea46e20 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py @@ -0,0 +1,195 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_NOT_QUOTE, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + accents, + shift_cardinal_gender, + strip_cardinal_apocope, +) + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos + + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross('negative: "true" ', "menos "), 0, 1) + self.optional_sign = optional_sign + + # Derivational strings append 'avo' as a suffix. Adding space for processing aid + fraction_stem = pynutil.insert(" avo") + plural = pynutil.insert("s") + conjunction = pynutil.insert(" y ") + + integer = ( + pynutil.delete('integer_part: "') + + strip_cardinal_apocope(pynini.closure(NEMO_NOT_QUOTE)) + + pynutil.delete('"') + ) + + numerator_one = pynutil.delete('numerator: "') + pynini.accep("un") + pynutil.delete('" ') + numerator = ( + pynutil.delete('numerator: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE), "un") + + pynutil.delete('" ') + ) + + denominator_add_stem = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + fraction_stem + pynutil.delete('" morphosyntactic_features: "add_root"') + ) + denominator_ordinal = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denominator_cardinal = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + ) + + denominator_singular = pynini.union(denominator_add_stem, denominator_ordinal) + if not deterministic: + # Occasional exceptions + denominator_singular |= denominator_add_stem @ pynini.string_map( + [("once avo", "undécimo"), ("doce avo", "duodécimo")] + ) + denominator_plural = denominator_singular + plural + + # Merging operations + merge = pynini.cdrewrite( + pynini.cross(" y ", "i"), "", "", NEMO_SIGMA + ) # The denominator must be a single word, with the conjunction "y" replaced by i + merge @= pynini.cdrewrite(delete_space, "", pynini.difference(NEMO_CHAR, "parte"), NEMO_SIGMA) + + # The merger can produce duplicate vowels. This is not allowed in orthography + delete_duplicates = pynini.string_map([("aa", "a"), ("oo", "o")]) # Removes vowels + delete_duplicates = pynini.cdrewrite(delete_duplicates, "", "", NEMO_SIGMA) + + remove_accents = pynini.cdrewrite( + accents, + pynini.union(NEMO_SPACE, pynini.accep("[BOS]")) + pynini.closure(NEMO_NOT_SPACE), + pynini.closure(NEMO_NOT_SPACE) + pynini.union("avo", "ava", "ésimo", "ésima"), + NEMO_SIGMA, + ) + merge_into_single_word = merge @ remove_accents @ delete_duplicates + + fraction_default = numerator + delete_space + insert_space + (denominator_plural @ merge_into_single_word) + + fraction_with_one = ( + numerator_one + delete_space + insert_space + (denominator_singular @ merge_into_single_word) + ) + + fraction_with_cardinal = strip_cardinal_apocope(numerator | numerator_one) + fraction_with_cardinal += ( + delete_space + pynutil.insert(" sobre ") + strip_cardinal_apocope(denominator_cardinal) + ) + + if not deterministic: + # There is an alternative rendering where ordinals act as adjectives for 'parte'. This requires use of the feminine + # Other rules will manage use of "un" at end, so just worry about endings + exceptions = pynini.string_map([("tercia", "tercera")]) + apply_exceptions = pynini.cdrewrite(exceptions, "", "", NEMO_SIGMA) + vowel_change = pynini.cdrewrite(pynini.cross("o", "a"), "", pynini.accep("[EOS]"), NEMO_SIGMA) + + denominator_singular_fem = shift_cardinal_gender(denominator_singular) @ vowel_change @ apply_exceptions + denominator_plural_fem = denominator_singular_fem + plural + + numerator_one_fem = shift_cardinal_gender(numerator_one) + numerator_fem = shift_cardinal_gender(numerator) + + fraction_with_cardinal |= ( + (numerator_one_fem | numerator_fem) + + delete_space + + pynutil.insert(" sobre ") + + shift_cardinal_gender(denominator_cardinal) + ) + + # Still need to manage stems + merge_stem = pynini.cdrewrite( + delete_space, "", pynini.union("avo", "ava", "avos", "avas"), NEMO_SIGMA + ) # For managing alternative spacing + merge_stem @= remove_accents @ delete_duplicates + + fraction_with_one_fem = numerator_one_fem + delete_space + insert_space + fraction_with_one_fem += pynini.union( + denominator_singular_fem @ merge_stem, + denominator_singular_fem @ merge_into_single_word, + ) # Both forms exists + fraction_with_one_fem += pynutil.insert(" parte") + fraction_with_one_fem @= pynini.cdrewrite( + pynini.cross("una media", "media"), "", "", NEMO_SIGMA + ) # "media" not "una media" + + fraction_default_fem = numerator_fem + delete_space + insert_space + fraction_default_fem += pynini.union( + denominator_plural_fem @ merge_stem, + denominator_plural_fem @ merge_into_single_word, + ) + fraction_default_fem += pynutil.insert(" partes") + + fraction_default |= ( + numerator + delete_space + insert_space + denominator_plural @ merge_stem + ) # Case of no merger + fraction_default |= fraction_default_fem + + fraction_with_one |= numerator_one + delete_space + insert_space + denominator_singular @ merge_stem + fraction_with_one |= fraction_with_one_fem + + fraction_with_one @= pynini.cdrewrite( + pynini.cross("un medio", "medio"), "", "", NEMO_SIGMA + ) # "medio" not "un medio" + + fraction = fraction_with_one | fraction_default | fraction_with_cardinal + graph_masc = optional_sign + pynini.closure(integer + delete_space + conjunction, 0, 1) + fraction + + # Manage cases of fem gender (only shows on integer except for "medio") + integer_fem = shift_cardinal_gender(integer) + fraction_default |= ( + shift_cardinal_gender(numerator) + + delete_space + + insert_space + + (denominator_plural @ pynini.cross("medios", "medias")) + ) + fraction_with_one |= ( + pynutil.delete(numerator_one) + delete_space + (denominator_singular @ pynini.cross("medio", "media")) + ) + + fraction_fem = fraction_with_one | fraction_default | fraction_with_cardinal + graph_fem = optional_sign + pynini.closure(integer_fem + delete_space + conjunction, 0, 1) + fraction_fem + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + self.graph = graph_masc | graph_fem + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..fc63a833e98ef73b72fe6380a65088febcdc4e43 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py @@ -0,0 +1,115 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_preserve_order, + delete_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ones +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv")) +unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv")) + +unit_singular_fem = pynini.project(unit_plural_fem, "input") +unit_singular_masc = pynini.project(unit_plural_masc, "input") + +unit_plural_fem = pynini.project(unit_plural_fem, "output") +unit_plural_masc = pynini.project(unit_plural_masc, "output") + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "dos" units: "gramos" } } -> "dos gramos" + measure { decimal { integer_part: "dos" quantity: "millones" units: "gramos" } } -> "dos millones de gramos" + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + graph_decimal_masc = decimal.delete_tokens(decimal.graph_masc) + graph_decimal_fem = decimal.delete_tokens(decimal.graph_fem) + graph_cardinal_masc = cardinal.delete_tokens(cardinal.graph_masc) + graph_cardinal_fem = cardinal.delete_tokens(cardinal.graph_fem) + graph_fraction_fem = fraction.delete_tokens(fraction.graph_fem) + graph_fraction_masc = fraction.delete_tokens(fraction.graph_masc) + + unit_masc = (unit_plural_masc | unit_singular_masc) + pynini.closure( + NEMO_WHITE_SPACE + "por" + pynini.closure(NEMO_NOT_QUOTE, 1), 0, 1 + ) + unit_masc |= "por" + pynini.closure(NEMO_NOT_QUOTE, 1) + unit_masc = pynutil.delete("units: \"") + (pynini.closure(NEMO_NOT_QUOTE) @ unit_masc) + pynutil.delete("\"") + + unit_fem = (unit_plural_fem | unit_singular_fem) + pynini.closure( + NEMO_WHITE_SPACE + "por" + pynini.closure(NEMO_NOT_QUOTE, 1), 0, 1 + ) + unit_fem = pynutil.delete("units: \"") + (pynini.closure(NEMO_NOT_QUOTE) @ unit_fem) + pynutil.delete("\"") + + graph_masc = (graph_cardinal_masc | graph_decimal_masc) + NEMO_WHITE_SPACE + unit_masc + graph_masc |= graph_fraction_masc + NEMO_WHITE_SPACE + pynutil.insert("de ") + unit_masc + graph_masc |= pynutil.add_weight( + graph_fraction_masc @ (NEMO_SIGMA + pynini.union("medio", "medios")) + NEMO_WHITE_SPACE + unit_masc, -0.001 + ) # "medio litro" not "medio de litro" + + graph_fem = (graph_cardinal_fem | graph_decimal_fem) + NEMO_WHITE_SPACE + unit_fem + graph_fem |= graph_fraction_fem + NEMO_WHITE_SPACE + pynutil.insert("de ") + unit_fem + graph_fem |= pynutil.add_weight( + graph_fraction_fem @ (NEMO_SIGMA + pynini.union("media", "medias")) + NEMO_WHITE_SPACE + unit_fem, -0.001 + ) + + graph = graph_masc | graph_fem + + graph = ( + pynini.cdrewrite( + pynutil.insert(" de"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph + ) # billones de xyz + + graph @= pynini.cdrewrite(pynini.cross(ones, "uno"), "", NEMO_WHITE_SPACE + "por", NEMO_SIGMA) + + # To manage alphanumeric combonations ("a-8, 5x"), we let them use a weighted default path. + alpha_num_unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph_alpha_num = pynini.union( + (graph_cardinal_masc | graph_decimal_masc) + NEMO_SPACE + alpha_num_unit, + alpha_num_unit + delete_extra_space + (graph_cardinal_masc | graph_decimal_masc), + ) + + math = pynutil.add_weight( + pynutil.delete("units: \"math\"") + delete_space + graph_cardinal_masc + delete_space, -1 + ) + + graph |= pynutil.add_weight(graph_alpha_num, 0.01) + graph |= math + + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..decb8a59f756df7da2b2159f43f773f308aff3a0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + +fem_singular = pynini.project(fem, "input") +masc_singular = pynini.project(masc, "input") + +fem_plural = pynini.project(fem, "output") +masc_plural = pynini.project(masc, "output") + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "euro" integer_part: "un"} -> "un euro" + money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un"} -> "uno coma cero cero uno euros" + money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true} -> "una libra cuarenta" + money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "peniques" preserve_order: true} -> "una libra con cuarenta peniques" + money { fractional_part: "un" currency_min: "penique" preserve_order: true} -> "un penique" + + Args: + decimal: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + maj_singular_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + maj_singular_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + maj_plural_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + maj_plural_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + maj_masc = maj_plural_masc | maj_singular_masc # Tagger kept quantity resolution stable + maj_fem = maj_plural_fem | maj_singular_fem + + min_singular_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + min_singular_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + min_plural_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + min_plural_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + min_masc = min_plural_masc | min_singular_masc + min_fem = min_plural_fem | min_singular_fem + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1) + + # *** currency_maj + graph_integer_masc = integer_part + NEMO_SPACE + maj_masc + graph_integer_fem = shift_cardinal_gender(integer_part) + NEMO_SPACE + maj_fem + + graph_integer = graph_integer_fem | graph_integer_masc + + # *** currency_maj + (***) | ((con) *** current_min) + graph_integer_with_minor_masc = ( + graph_integer_masc + + NEMO_SPACE + + pynini.union( + optional_add_and + strip_cardinal_apocope(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) # Could be minor currency that is different gender + + delete_preserve_order + ) + + graph_integer_with_minor_fem = ( + graph_integer_fem + + NEMO_SPACE + + pynini.union( + optional_add_and + shift_cardinal_gender(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) # Could be minor currency that is different gender + + delete_preserve_order + ) + + graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc + + ## *** coma *** currency_maj + graph_decimal_masc = decimal.graph_masc + NEMO_SPACE + maj_masc + + graph_decimal_fem = decimal.graph_fem + graph_decimal_fem |= decimal.numbers_only_quantity # can still have "x billions" with fem currency + graph_decimal_fem += NEMO_SPACE + maj_fem + + graph_decimal = graph_decimal_fem | graph_decimal_masc + graph_decimal = ( + pynini.cdrewrite( + pynutil.insert(" de"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph_decimal + ) # formally it's millones/billones de *** + + # *** current_min + graph_minor_masc = fractional_part + NEMO_SPACE + min_masc + delete_preserve_order + graph_minor_fem = shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem + delete_preserve_order + + graph_minor = graph_minor_fem | graph_minor_masc + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..e25479f86fbaa80d90a5e95ad7c428d5efa76646 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_number_gender + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals + e.g. ordinal { integer: "tercer" } } -> "tercero" + -> "tercera" + -> "tercer" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # masculne gender we leave as is + graph_masc = graph + pynutil.delete(" morphosyntactic_features: \"gender_masc") + + # shift gender + graph_fem_ending = graph @ pynini.cdrewrite( + pynini.cross("o", "a"), "", NEMO_SPACE | pynini.accep("[EOS]"), NEMO_SIGMA + ) + graph_fem = shift_number_gender(graph_fem_ending) + pynutil.delete(" morphosyntactic_features: \"gender_fem") + + # Apocope just changes tercero and primero. May occur if someone wrote 11.er (uncommon) + graph_apocope = ( + pynini.cross("tercero", "tercer") + | pynini.cross("primero", "primer") + | pynini.cross("undécimo", "decimoprimer") + ) # In case someone wrote 11.er with deterministic + graph_apocope = (graph @ pynini.cdrewrite(graph_apocope, "", "", NEMO_SIGMA)) + pynutil.delete( + " morphosyntactic_features: \"apocope" + ) + + graph = graph_apocope | graph_masc | graph_fem + + if not deterministic: + # Plural graph + graph_plural = pynini.cdrewrite( + pynutil.insert("s"), pynini.union("o", "a"), NEMO_SPACE | pynini.accep("[EOS]"), NEMO_SIGMA + ) + + graph |= (graph @ graph_plural) + pynutil.delete("/plural") + + self.graph = graph + pynutil.delete("\"") + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..0156ce30979f0a84a451e576a7629e2538e89f95 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { number_part: "uno dos tres uno dos tres cinco seis siete ocho" } + -> uno dos tres uno dos tres cinco seis siete ocho + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize") + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() + + +# number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + +# delete_tokens = self.delete_tokens(number_part) +# self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..f1adb25735ec6ece8e5d3ebdc0d23b5618cb2e2d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py @@ -0,0 +1,257 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +alt_minutes = pynini.string_file(get_abs_path("data/time/alt_minutes.tsv")) + +morning_times = pynini.string_file(get_abs_path("data/time/morning_times.tsv")) +afternoon_times = pynini.string_file(get_abs_path("data/time/afternoon_times.tsv")) +evening_times = pynini.string_file(get_abs_path("data/time/evening_times.tsv")) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "doce" minutes: "media" suffix: "a m" } -> doce y media de la noche + time { hours: "doce" } -> twelve o'clock + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + change_minutes = pynini.cdrewrite(alt_minutes, pynini.accep("[BOS]"), pynini.accep("[EOS]"), NEMO_SIGMA) + + morning_phrases = pynini.cross("am", "de la mañana") + afternoon_phrases = pynini.cross("pm", "de la tarde") + evening_phrases = pynini.cross("pm", "de la noche") + + # For the 12's + mid_times = pynini.accep("doce") + mid_phrases = ( + pynini.string_map([("pm", "del mediodía"), ("am", "de la noche")]) + if deterministic + else pynini.string_map( + [ + ("pm", "de la mañana"), + ("pm", "del día"), + ("pm", "del mediodía"), + ("am", "de la noche"), + ("am", "de la medianoche"), + ] + ) + ) + + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = (minute @ change_minutes) if deterministic else pynini.union(minute, minute @ change_minutes) + + suffix = ( + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_hms = ( + hour + + pynutil.insert(" horas ") + + delete_space + + minute + + pynutil.insert(" minutos y ") + + delete_space + + second + + pynutil.insert(" segundos") + ) + + graph_hm = hour + delete_space + pynutil.insert(" y ") + minute + graph_hm |= pynini.union( + (hour @ morning_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ morning_phrases), + (hour @ afternoon_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ afternoon_phrases), + (hour @ evening_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ evening_phrases), + (hour @ mid_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ mid_phrases), + ) + + graph_h = pynini.union( + hour, + (hour @ morning_times) + delete_space + insert_space + (suffix @ morning_phrases), + (hour @ afternoon_times) + delete_space + insert_space + (suffix @ afternoon_phrases), + (hour @ evening_times) + delete_space + insert_space + (suffix @ evening_phrases), + (hour @ mid_times) + delete_space + insert_space + (suffix @ mid_phrases), + ) + + graph = (graph_hms | graph_hm | graph_h) + optional_zone + + if not deterministic: + graph_style_1 = pynutil.delete(" style: \"1\"") + graph_style_2 = pynutil.delete(" style: \"2\"") + + graph_menos = hour + delete_space + pynutil.insert(" menos ") + minute + graph_style_1 + graph_menos |= ( + (hour @ morning_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ morning_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ afternoon_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ afternoon_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ evening_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ evening_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ mid_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ mid_phrases) + + graph_style_1 + ) + graph_menos += optional_zone + + graph_para = minute + pynutil.insert(" para las ") + delete_space + hour + graph_style_2 + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ morning_times) + + delete_space + + insert_space + + (suffix @ morning_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ afternoon_times) + + delete_space + + insert_space + + (suffix @ afternoon_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ evening_times) + + delete_space + + insert_space + + (suffix @ evening_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ mid_times) + + delete_space + + insert_space + + (suffix @ mid_phrases) + + graph_style_2 + ) + graph_para += optional_zone + graph_para @= pynini.cdrewrite( + pynini.cross(" las ", " la "), "para", "una", NEMO_SIGMA + ) # Need agreement with one + + graph |= graph_menos | graph_para + delete_tokens = self.delete_tokens(graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..b62fa89676fef1ae9dc9a0e0c92b8b6bd10b9345 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py @@ -0,0 +1,73 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.decimals import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + time_graph = TimeFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + | measure_graph + | decimal_graph + | ordinal_graph + | date_graph + | electronic_graph + | money_graph + | fraction_graph + | whitelist_graph + | telephone_graph + | time_graph + ) + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..88cbf6b48e85b52117c53a80560c8fa0a7aa4043 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"es_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6127bea93f18d3233067a12d4b25580a82dba1b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv @@ -0,0 +1,8 @@ +20s twenties +30s thirties +40s forties +50s fifties +60s sixties +70s seventies +80s eighties +90s nineties \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..98a4e7d5d5b6d6aceee049a7a315abb1e21f88ea --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 janvier +2 février +3 mars +4 avril +5 mai +6 juin +7 juillet +8 août +9 septembre +10 octobre +11 novembre +12 décembre \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb199174ca7990b19cf3b532b01367573e3d2a94 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv @@ -0,0 +1,7 @@ +premième tout +deuxième demi +deuxièmes demi +troisième tiers +troisièmes tiers +quatrième quart +quatrièmes quarts \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..55f0d46435462504b4a87f87c88c4db4c44ad3da --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv @@ -0,0 +1,17 @@ +m mètres +m² mètres carrés +m³ mètres cubes +s secondes +min minutes +h heures +° degrés +°C degrés celsius +g grammes +l litres +kg kilos +'' pouce +lb livres +% pour cent +‰ pour mille +km/h kilomètres heure +m/h mètres à l’heure \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4f5375b2940d04557fb2964776581151491930b0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 un +2 deux +3 trois +4 quatre +5 cinq +6 six +7 sept +8 huit +9 neuf \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bdb07292f323e72e48339646bfc3271dd18c1a56 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv @@ -0,0 +1 @@ +8 quatre-vingt \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b00412f6d033c1597ef828ef52e4faa33f2dc08e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv @@ -0,0 +1 @@ +9 quatre-vingt \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a17792e406a6f9feb5977fe25981542e64ddf1b4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv @@ -0,0 +1,10 @@ +million +millions +milliard +milliards +billion +billions +billiard +billiards +trillion +trillions \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv new file mode 100644 index 0000000000000000000000000000000000000000..80c4116f7c7f93687b76bf5bb47d6160a565dc75 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv @@ -0,0 +1 @@ +7 soixante \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c22d6879ea19c67cde0d44352550844880f728aa --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv @@ -0,0 +1,9 @@ +11 onze +12 douze +13 treize +14 quatorze +15 quinze +16 seize +17 dix-sept +18 dix-huit +19 dix-neuf \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e7e79c1397e06c478ca09ec165efb337c69e06de --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv @@ -0,0 +1 @@ +10 dix \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b72cf3dd8e0a7a441148bdf4527c6e08eed4ff0a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv @@ -0,0 +1,5 @@ +2 vingt +3 trente +4 quarante +5 cinquante +6 soixante \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2af43a7086f918593571d14ad147de530e9b15af --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zéro \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2d21198f3ee1e88f8a50bca59705c537b3c02138 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv @@ -0,0 +1,30 @@ +quatre quatrième +cinq cinquième +neuf neuvième +onze onzième +douze douzième +treize treizième +quatorze quatorzième +quinze quinzième +seize seizième +dix-neuf dix neuvième +trente trentième +quarante quarantième +cinquante cinquantième +soixante soixantième +soixante-onze soixante onzième +soixante-douze soixante douzième +soixante-treize soixante treizième +soixante-quatorze soixante quatorzième +soixante-quinze soixante quinzième +soixante-seize soixante seizième +soixante-dix-neuf soixante dix neuvième +quatre-vingt-onze quatre vingt onzième +quatre-vingt-douze quatre vingt douzième +quatre-vingt-treize quatre vingt treizième +quatre-vingt-quatorze quatre vingt quatorzième +quatre-vingt-quinze quatre vingt quinzième +quatre-vingt-seize quatre vingt seizième +quatre-vingt-dix-neuf quatre vingt dix neuvième +cents centième +mille millième \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv new file mode 100644 index 0000000000000000000000000000000000000000..73adb8e043daca73a33a791255f8f64063806ace --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv @@ -0,0 +1,6 @@ +e ième +ème ième +° ième +er ier +ère ière +es ièmes \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..dc563bdab6a3b86a2a41bc9e24be837a15df13d7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv @@ -0,0 +1,13 @@ +Mᵐᵉ madame +Mᵐᵉˢ mesdames +Mˡˡᵉ mademoiselle +Mˡˡᵉˢ mademoiselles +Dʳ docteur +Dʳˢ docteurs +Dʳᵉ docteure +Dʳᵉˢ docteures +apr. J.-C. après jésus-christ +av. J.-C. avant Jésus-Christ +le hon. l’honorable +le très hon. le très hononrable +% pour cent \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..97266a4cb1e16de5faaf0226f0a80205a4bae3f0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "mille" } + "2,000,000" -> cardinal { integer: "deux millions" } + This grammar covers from single digits to hundreds of billions ("milliards" in French). + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Single digits + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + digits = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) # 1 to 9 + digits_no_one = (NEMO_DIGIT - "1") @ digits + one = "1" @ digits + + # Double digits + ten = pynini.string_file(get_abs_path("data/numbers/ten.tsv")) + teens = pynini.string_file(get_abs_path("data/numbers/teens.tsv")) # 11 to 19 + teens_no_one = (pynini.accep("1") + (NEMO_DIGIT - "1")) @ teens + + # Simple tens + tens = pynini.string_file(get_abs_path("data/numbers/tens_simple.tsv")) + ties_simple_unit = pynutil.insert("-") + digits_no_one + ties_simple_one = insert_space + pynini.cross("1", "et un") + ties_simple = tens + (pynutil.delete("0") | (ties_simple_unit | ties_simple_one)) # 20 to 69 + + # Complex tens + seventy = pynini.string_file(get_abs_path("data/numbers/seventy.tsv")) + seventy_unit = pynutil.insert("-") + ((pynutil.insert("1") + NEMO_DIGIT) @ teens_no_one) + seventy_one = insert_space + pynini.cross("1", "et onze") + seventies = seventy + (pynini.cross("0", "-dix") | (seventy_unit | seventy_one)) # 70 to 79 + + eighty = pynini.string_file(get_abs_path("data/numbers/eighty.tsv")) + eighties = eighty + (pynini.cross("0", "s") | (pynutil.insert("-") + digits)) # 80 to 89 + + ninety = pynini.string_file(get_abs_path("data/numbers/ninety.tsv")) + nineties_unit = pynutil.insert("-") + ((pynutil.insert("1") + NEMO_DIGIT) @ teens) + nineties = ninety + (pynini.cross("0", "-dix") | nineties_unit) # 90 to 99 + + all_double_digits = ten | teens | ties_simple | seventies | eighties | nineties + self.all_double_digits = all_double_digits + + one_to_all_tens = digits | all_double_digits + self.one_to_all_tens = one_to_all_tens.optimize() + + # Hundreds + hundreds_parts = (pynutil.delete("0") + insert_space + digits) | (insert_space + all_double_digits) + one_hundreds = pynini.cross("1", "cent") + (pynutil.delete("00") | hundreds_parts) + multiple_hundreds = (digits_no_one + insert_space + pynutil.insert("cent")) + ( + pynini.cross("00", "s") | hundreds_parts + ) + all_hundreds = one_hundreds | multiple_hundreds + self.all_hundreds = all_hundreds + + # Thousands + delete_separator = pynini.closure(pynutil.delete(" "), 0, 1) + one_thousand = pynini.cross("1", "mille") + delete_separator + other_thousands = ( + (digits_no_one | all_double_digits | all_hundreds) + + insert_space + + pynutil.insert("mille") + + delete_separator + ) + all_thousands = ( + ((one_thousand | other_thousands) + pynutil.delete("000")) + | (one_thousand + pynutil.delete("00") + insert_space + (pynini.cross("1", "et un") | digits_no_one)) + | (other_thousands + pynutil.delete("00") + insert_space + digits) + | ((one_thousand | other_thousands) + pynutil.delete("0") + insert_space + all_double_digits) + | ((one_thousand | other_thousands) + insert_space + all_hundreds) + ) + + # Millions + digits_to_hundreds_no_one = digits_no_one | all_double_digits | all_hundreds + digits_to_thousands_no_one = digits_no_one | all_double_digits | all_hundreds | all_thousands + millions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 2) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 6) + all_millions = ( + one + insert_space + pynutil.insert("million") + | (digits_to_hundreds_no_one + insert_space + pynutil.insert("millions")) + ) + ( + millions_components + @ ( + delete_zeros + + pynini.closure(insert_space + (digits_to_thousands_no_one | pynini.cross("1", "et un")), 0, 1) + ) + ) + + # Billions + digits_to_millions_no_one = digits_to_thousands_no_one | all_millions + billions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 3) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 9) + all_billions = ( + one + insert_space + pynutil.insert("milliard") + | (digits_to_hundreds_no_one + insert_space + pynutil.insert("milliards")) + ) + ( + billions_components + @ ( + delete_zeros + + pynini.closure(insert_space + (digits_to_millions_no_one | pynini.cross("1", "et un")), 0, 1) + ) + ) + + # All Numbers Union + final_graph_masc = zero | one_to_all_tens | all_hundreds | all_thousands | all_millions | all_billions + + # Adding adjustment for fem gender (choice of gender will be random) + final_graph_fem = final_graph_masc | ( + final_graph_masc @ (pynini.closure(NEMO_SIGMA) + pynini.cross("un", "une")) + ) + final_graph = final_graph_fem | final_graph_masc + + self.all_nums_no_tokens = final_graph + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..ace0503795403b9ab85b00f6001a11fa9aea9507 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py @@ -0,0 +1,97 @@ +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + +# TODO: add articles? 'le...' + +month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv")) +eras = pynini.string_file(get_abs_path("data/dates/eras.tsv")) +delete_leading_zero = ( + pynutil.delete("0") | (NEMO_DIGIT - "0") +) + NEMO_DIGIT # reminder, NEMO_DIGIT = filter on digits + + +class DateFst(GraphFst): + '''Finite state transducer for classyfing dates, e.g.: + '02.03.2003' -> date {day: 'deux' month: 'mai' year: 'deux mille trois' preserve order: true} + ''' + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="dates", kind="classify") + + cardinal_graph = cardinal.all_nums_no_tokens + + # 'le' -> 'le', 'les' -> 'les' + le_determiner = pynini.accep("le ") | pynini.accep("les ") + self.optional_le = pynini.closure(le_determiner, 0, 1) + + # '01' -> 'un' + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + valid_day_number = pynini.union(*[str(x) for x in range(1, 32)]) + premier = pynini.string_map([("1", "premier")]) + day_number_to_word = premier | cardinal_graph + + digit_to_day = self.optional_le + optional_leading_zero @ valid_day_number @ day_number_to_word + self.day_graph = pynutil.insert("day: \"") + digit_to_day + pynutil.insert("\"") + + # '03' -> 'mars' + normalize_month_number = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = month_numbers.optimize() + month_graph = normalize_month_number @ number_to_month + self.month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + + # 2025 -> deux mille vingt cinq + accept_year_digits = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) + digits_to_year = accept_year_digits @ cardinal_graph + self.year_graph = pynutil.insert("year: \"") + digits_to_year + pynutil.insert("\"") + + # Putting it all together + self.fst = pynini.accep("") + + for separator in ["/", ".", "-"]: + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynutil.delete(separator) + + pynutil.insert(" ") + + self.month_graph + + pynini.closure(pynutil.delete(separator) + pynutil.insert(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + # Accepts "janvier", "février", etc + month_name_graph = pynutil.insert("month: \"") + month_numbers.project("output") + pynutil.insert("\"") + + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true}") + ) + + # Accepts "70s", "80s", etc + self.fst |= pynutil.insert("date { year: \"") + eras + pynutil.insert("\" preserve_order: true }") + + # Accepts date ranges, "17-18-19 juin" -> date { day: "17" day: "18": day: "19"} + for separator in ["-", "/"]: + day_range_graph = ( + pynutil.insert("day: \"") + + pynini.closure(digit_to_day + pynutil.delete(separator) + pynutil.insert(" "), 1) + + digit_to_day + + pynutil.insert("\"") + ) + + self.fst |= ( + pynutil.insert("date { ") + + day_range_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + self.fst = self.fst.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..1507d70c6eea85e7b1d7a260b8ed1e5c37153eca --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py @@ -0,0 +1,116 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 2 millions -> integer_part: "deux" quantity: "millions" + e.g. 2,4 millions -> integer_part: "deux" fractional_part: "quatre" quantity: "millions" + Args: + decimal_graph: DecimalFST + cardinal_graph: CardinalFST + """ + delete_separator = pynini.closure(pynutil.delete(" "), 0, 1) + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(delete_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -11,406 millions -> decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } + 114 billions -> decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.all_hundreds, cardinal.one_to_all_tens) + graph += pynini.closure(insert_space + graph) + + else: + # General pattern is 1-3 digits: map as cardinal, default to tens followed by digits otherwise \ + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.all_double_digits + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.all_hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.all_double_digits + + pynini.closure(insert_space + cardinal.all_double_digits, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,001" + ) + + graph = graph.optimize() + + delete_separator = pynutil.delete(",") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + graph_fractional = pynutil.insert('fractional_part: "') + graph + pynutil.insert('"') + + integers = cardinal.all_nums_no_tokens + graph_integer = pynutil.insert('integer_part: "') + integers + pynutil.insert('"') + final_graph_wo_sign = graph_integer + delete_separator + insert_space + graph_fractional + + final_graph_wo_negative = final_graph_wo_sign | get_quantity(final_graph_wo_sign, integers).optimize() + self.final_graph_wo_negative = final_graph_wo_negative + + final_graph = optional_graph_negative + final_graph_wo_negative + self.final_graph = final_graph + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..865ba6c06340a63a41d4b2db7f13648491f53d22 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "-4 1/3" -> + tokens { fraction { negative: "true" integer_part: "quatre" numerator: "un" denominator: "trois" morphosyntactic_features: "ième" } } + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction will be generated (used for audio-based normalization) - TBD + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinals = cardinal.all_nums_no_tokens + sing_numerator = pynini.accep("1") @ cardinals + pl_numerators = (pynini.closure(NEMO_DIGIT) - "1") @ cardinals + + add_denom_suffix = pynini.closure(NEMO_DIGIT) + pynutil.insert("e") + denominators = add_denom_suffix @ ordinal.graph + change_denom_label = pynini.cross("integer", "denominator") + pluralize_denom = pynini.closure(NEMO_SIGMA) + pynini.cross("\"ième\"", "\"ièmes\"") + + sing_fraction_graph = ( + pynutil.insert("numerator: \"") + + sing_numerator + + pynutil.insert("\" ") + + pynutil.delete("/") + + (denominators @ (change_denom_label + pynini.closure(NEMO_SIGMA))) + ) + + pl_fraction_graph = ( + pynutil.insert("numerator: \"") + + pl_numerators + + pynutil.insert("\" ") + + pynutil.delete("/") + + (denominators @ (change_denom_label + pluralize_denom)) + ) + + integer_part = pynutil.insert("integer_part: \"") + cardinals + pynutil.insert("\"") + optional_integer_part = pynini.closure(integer_part + pynini.accep(" "), 0, 1) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + optional_integer_part + (sing_fraction_graph | pl_fraction_graph) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..eae07a0267c2b34ea20f19cd914ae2dfe205fb5e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + "2e" -> ordinal { integer: "deux" morphosyntactic_features: "ième" } + This grammar covers from single digits to hundreds of billions ("milliardième" in French). + This FST also records the ending of the ordinal (called "morphosyntactic_features"). + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify") + + numbers = cardinal.all_nums_no_tokens + numbers_graph = pynutil.insert("integer: \"") + numbers + pynutil.insert("\"") + + suffixes = pynini.string_file(get_abs_path("data/ordinals/suffixes.tsv")) + suffixes_graph = pynutil.insert("morphosyntactic_features: \"") + suffixes + pynutil.insert("\"") + + final_graph = numbers_graph + pynutil.insert(NEMO_SPACE) + suffixes_graph + self.graph = final_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..0180c721b4fa38c1ef4269356d3402f2bb652dac --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py @@ -0,0 +1,126 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.decimals import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State aRchive (FAR) File. + More details to deployment at NeMo-text-processing/tools/text_processing_deployment. + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_fr_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic) + ordinal_graph = self.ordinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) + fraction_graph = self.fraction.fst + word_graph = WordFst(deterministic=deterministic).fst + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + self.date = DateFst(self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(word_graph, 200) + ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct), + 1, + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..c1fade95d359271ca93362863ad05033ee59dba9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "sr." -> tokens { name: "bonjour" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measures/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..07ebb473073bb4dd232dcfe9ed0899f7abb04c9a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. bonjour -> tokens { name: "bonjour" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7523e576221aa4246c3579a56c8f3094b846f7d7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/utils.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path) + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..a5ec9d163ce99d8092bbddc9b30eb08d1fe9cb69 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { negative: "true" integer: "un milliard et un" } -> "moins un milliard et un" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "moins") + insert_space, 0, 1) + number = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + final_graph = optional_sign + number + + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..b8e24d84a1389a93602199bd601f5e6169db8921 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date {day: "deux" month: "mars" year: "deux mille trois" preserve_order: true} -> deux mars deux mille trois + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + decade = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph_dmy = day + NEMO_SPACE + month + pynini.closure(NEMO_SPACE + year, 0, 1) + delete_preserve_order + graph_my = month + NEMO_SPACE + year + delete_preserve_order + graph_decade = decade + delete_preserve_order + + self.graph = graph_dmy | graph_my | graph_decade + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..5872bc439ac0de1ceedb4f239f7f56192078f345 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } -> moins onze virgule quatre cent six millions + decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } -> cent quatorze billions + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "moins ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" virgule ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..675ddefc557c52a07b63e80158f0cdccc1c55442 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py @@ -0,0 +1,58 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + denominator = ordinal.graph + irregular_denominators = pynini.string_file(get_abs_path("data/fractions/irregular_denominators.tsv")) + denominator = pynini.union(pynutil.add_weight(denominator @ irregular_denominators, -0.01), denominator) + denominator = (pynini.cross("denominator:", "integer:") + pynini.closure(NEMO_SIGMA)) @ denominator + + irregular_half = pynini.closure(NEMO_SIGMA) + pynini.cross("et un demi", "et demi") + + optional_integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + optional_integer = pynini.closure( + optional_integer + pynini.accep(NEMO_SPACE) + pynutil.insert("et") + insert_space, 0, 1 + ) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "moins") + insert_space, 0, 1) + + final_graph = optional_sign + optional_integer + numerator + pynini.accep(NEMO_SPACE) + denominator + final_graph = pynini.union(pynutil.add_weight(final_graph @ irregular_half, -0.01), final_graph) + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..c74c2ac9ad23bec3aff9728b438debdb40df225a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals + e.g. ordinal { integer: "deux" morphosyntactic_features: "ième" } -> "deuxième" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, strip_dashes: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + ones = pynini.cross("un", "prem") + + irregular_numbers = pynini.string_file(get_abs_path("data/ordinals/irregular_numbers.tsv")) + irregular_numbers = pynini.closure(pynini.closure(NEMO_NOT_QUOTE, 1) + NEMO_SPACE) + irregular_numbers + exceptions = pynini.project(irregular_numbers, "input") + exception_suffix = ( + pynutil.delete(" morphosyntactic_features: \"ième") + + pynini.closure(pynini.accep("s"), 0, 1) + + pynutil.delete("\"") + ) + irregular_numbers_graph = ( + pynutil.delete("integer: \"") + irregular_numbers + pynutil.delete("\"") + exception_suffix + ) + + numbers = pynini.closure(NEMO_NOT_QUOTE, 1) + numbers = pynini.difference(numbers, exceptions) + + if strip_dashes: + remove_dashes = pynini.closure(NEMO_ALPHA, 1) + pynini.cross("-", " ") + pynini.closure(NEMO_ALPHA, 1) + remove_dashes = pynini.closure(remove_dashes, 0) + numbers = pynini.union(numbers, pynutil.add_weight(numbers @ remove_dashes, -0.0001)) + + regular_ordinals = pynini.union(numbers, pynutil.add_weight(ones, -0.0001)) + regular_ordinals_graph = ( + pynutil.delete("integer: \"") + + regular_ordinals + + pynutil.delete("\"") + + pynutil.delete(" morphosyntactic_features: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + final_graph = pynini.union(regular_ordinals_graph, irregular_numbers_graph) + self.graph = final_graph + + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..e289e3d0ef7263e9f6330faeea66235cddfcf303 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.decimals import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.ordinal import OrdinalFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo-text-processing/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(ordinal=ordinal, deterministic=deterministic) + fraction_graph = fraction.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst + + graph = cardinal_graph | decimal_graph | ordinal_graph | fraction_graph | whitelist_graph | date_graph + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..8d478e5834609d70368649fb99dd751502a6e497 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"fr_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logger.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4cf3338291c4a198f33ae65944038b6b99b413f6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c5c26d736702cae7cdddbbf7a9d5e5704707e2ce --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv @@ -0,0 +1,14 @@ +.com punto com +.org punto org +.gov punto gov +.uk punto UK +.fr punto FR +.net punto net +.html punto html +.py punto python +.ru punto RU +.us punto US +.de punto DE +.it punto IT +.jpg punto jpeg +.edu punto edu \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a0c7562948684993372a2c389c9f32297550e5f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..56c9270c93111a58019ffa5f1141d30623c1fbd1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- trattino +_ trattino basso +! punto esclamativo +# cancelletto +$ dollaro +% percento +& e commerciale +' apostrofo +* asterisco ++ più +/ slash += uguale +? punto interrogativo +^ esponente +` accento +{ apri graffa +| barra verticale +} chiudi graffa +~ tilde +, virgola +: due punti diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb359e6a0bd8d46a68708aed1eb0ff782c089ec9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv @@ -0,0 +1,64 @@ +d giorno +f fahrenheit +c celsius +°C grado celsius +°F grado fahrenheit +K kelvin +km chilometro +m metro +cm centimetro +mm millimetro +μm micrometro +nm nanometro +dm decimetro +pm picometro +hm ettometro +ha ettaro +mi miglio +m² metro quadrato -0.0001 +km² chilometro quadrato -0.0001 +m2 metro quadrato +km2 chilometro quadrato +m3 metro cubo +ft foot +g grammo +µg microgrammo +mg milligrammo +kg chilogrammo +lb libbra +h ora +s secondo +min minuto +ms millisecondo +μs microsecondo +hz hertz +kw kilowatt +ghz gigahertz +khz kilohertz +mhz megahertz +v volt +mc megacoulomb +mA milliampere +A ampere +tw terawatt +mv millivolt +mw megawatt +gw gigawatt +ω ohm +db decibel +gb gigabyte +kb kilobit +pb petabit +mb megabyte +kb kilobyte +tb terabyte +kv kilovolt +mv megavolt +kn kilonewton +ml millilitro +l litro +dl decilitro +bar bar +kcal chilocaloria +cal caloria +% percento \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv new file mode 100644 index 0000000000000000000000000000000000000000..31d2e1975e93839929cfa9bf09902e88ae5286c2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv @@ -0,0 +1,13 @@ +grado celsius gradi celsius +metro metri +ora ore +grammo grammi +secondo secondi +minuto minuti +chilometro chilometri +bit bits +byte bytes +caloria calorie +miglio miglia +litro litri +giorno giorni \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..93a991e4344abe01d81b8c79c7a6397f2f4b2e15 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv @@ -0,0 +1,11 @@ +€ euro +$ dollaro +£ sterlina +usd dollaro statunitense +eur euro +cad dollaro canadese +chf franco svizzero +hkd dollaro di hong kong +¥ yen +rs rupia +₽ rublo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv new file mode 100644 index 0000000000000000000000000000000000000000..59c91fff9c2768071367089d643d2ef8597d0557 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv @@ -0,0 +1,11 @@ +€ centesimo +$ centesimo +£ penny +usd centesimo +eur centesimo +cad centesimo +chf centesimo +hkd centesimo +¥ yen +rs paisa +₽ copeca diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..542c49d1dce1a405c630f71cbe45cfa6e0df8721 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv @@ -0,0 +1,2 @@ +sterlina sterline +rupia rupie diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b3905f693cf01d8949afa4e304f5cd5e36ae08b2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv @@ -0,0 +1,10 @@ +centesimo centesimi +euro euro +dollaro dollari +penny penny +dollaro canadese dollari canadesi +franco svizzero franchi svizzeri +dollaro statunitense dollari statunitensi +rublo rubli +dollaro di hong hong dollari di hong kong +copeca copeche diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4f2bf34e405925de3c2c4d2c29e95068c49a03bf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +uno 1 +due 2 +tre 3 +quattro 4 +cinque 5 +sei 6 +sette 7 +otto 8 +nove 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..20f4aa2d16907a70b8dad16a8111f27251d1bf12 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +duecento 2 +trecento 3 +quattrocento 4 +cinquecento 5 +seicento 6 +settecento 7 +ottocento 8 +novecento 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fac386746f3e73498eab9015b0997669f58f2425 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv @@ -0,0 +1,2 @@ +milioni +miliardi \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5c256ec6edb33d1002afb2c4796a06b9fb78aeac --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +dieci 10 +undici 11 +dodici 12 +tredici 13 +quattordici 14 +quindici 15 +sedici 16 +diciassette 17 +diciotto 18 +diciannove 19 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5637114c26793f0b65549322038bdf63c2748884 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +venti 2 +trenta 3 +quaranta 4 +cinquanta 5 +sessanta 6 +settanta 7 +ottanta 8 +novanta 9 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1cd57016368caeab4f9453ce692c904d9c51747f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv @@ -0,0 +1,8 @@ +ventotto 28 +trentotto 38 +quarantotto 48 +cinquantotto 58 +sessantotto 68 +settantotto 78 +ottantotto 88 +novantotto 98 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3ffbcec03d076193a169bf8539b1ae9464185840 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv @@ -0,0 +1,8 @@ +ventuno 21 +trentuno 31 +quarantuno 41 +cinquantuno 51 +sessantuno 61 +settantuno 71 +ottantuno 81 +novantuno 91 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c479272d40391d642dfa768cfa16be064e42ae5c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..15d65d81d2c081612e319f9a370fedf4bbe4bff5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv @@ -0,0 +1,65 @@ +dr dottor +dr. dottor +Dr. dottor +dott. dottor +Dott. dottor +dott.sa dottoressa +Dott.sa dottoressa +prof. professore +Prof. professore +proff. professori +Proff. professori +prof.ssa professoressa +Prof.ssa professoressa +Sig. signore +Sig.na signorina +sig. signore +sig.na signorina +sig signore +Sigg. signori +sigg. signori +Sig.ri signori +sig.ri signori +Sig.re signore +sig.re signore +Sig.ra signora +sig.ra signora +Sig.re signorine +sig.re signorine +Avv. avvocato +avv. avvocato +Arch. architetto +arch. architetto +Rag. ragioniere +rag. ragioniere +Geom. geometra +geom. geometra +Ing. ingegnere +ing. ingegnere +Ingg. ingegneri +ingg. ingegneri +Fam. famiglia +fam. famiglia +f.lli fratelli +F.lli fratelli +Rev. reverendo +On. onorevole +on. onorevole +Onn. onorevoli +onn. onorevoli +Min. ministro +Sen. senatore +c.v.d. come volevasi dimostrare +ecc. eccetera +ecc eccetera +v.le viale +p.za piazza +l.go largo +loc. località +ist. istituto +segr. segreteria +set. settore +uff. ufficio +dip. dipartimento +div. divisione ++ più diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..9b3cbce536a73f4d68513316178bbba89d4103f8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py @@ -0,0 +1,183 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) +tens = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) +tens_one = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_one.tsv"))) +tens_eight = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_eight.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils) + and converts to a string of digits: + "1 000" -> "1000" + "1.000.000" -> "1000000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +class CardinalFst(GraphFst): + ''' + Finite state transducer for classifying cardinals in Italian, e.g. + "1000" -> cardinal { integer: "mille" } + "2.000.000" -> cardinal { integer: "due milioni" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + ''' + + def __init__(self, deterministic: bool = True): + super().__init__(name='cardinal', kind='classify', deterministic=deterministic) + + # single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + + # double digit + graph_tens = teen + graph_tens |= tens + (pynutil.delete('0') | graph_digit) + graph_tens |= pynutil.add_weight(tens_one, -0.01) + graph_tens |= pynutil.add_weight(tens_eight, -0.01) + + self.tens = graph_tens.optimize() + + self.two_digit_no_zero = pynini.union( + graph_digit, graph_tens, (pynini.cross('0', NEMO_SPACE) + graph_digit) + ).optimize() + + # three digit + graph_hundreds = hundreds + pynini.union( + pynutil.delete('00'), insert_space + graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit) + ) + graph_hundreds |= ( + pynini.cross('1', 'cento') + + insert_space + + pynini.union(graph_tens, pynutil.delete("0") + graph_digit, pynutil.delete("00")) + ) + + self.hundreds = graph_hundreds.optimize() + + # three digit leading zeros + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit + ) + + graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + + # thousands + graph_thousands_component_at_least_one_none_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert("mila") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mille") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert("mila") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mille") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + # higher + graph_million = pynutil.add_weight(pynini.cross("000001", "un milione"), -0.001) + graph_million |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" milioni") + graph_million |= pynutil.delete("000000") + graph_million += insert_space + + graph_billion = pynutil.add_weight(pynini.cross("000001", "un miliardo"), -0.001) + graph_billion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" miliardi") + graph_billion |= pynutil.delete("000000") + graph_billion += insert_space + + graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trilione"), -0.001) + graph_trillion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" trilioni") + graph_trillion |= pynutil.delete("000000") + graph_trillion += insert_space + + graph = ( + graph_trillion + + graph_billion + + graph_million + + (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000")) + ) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + ) + + self.graph |= zero + + self.graph = filter_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..fee715fe37bf40c1762345135e389e7962b62ece --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py @@ -0,0 +1,121 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + cardinal_separator, + decimal_separator, + strip_cardinal_apocope, +) +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ """ + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -2 milioni: decimal { negative: "true" integer_part: "due" quantity: "milioni" preserve_order: true } --> + meno due milioni + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.hundreds, cardinal.tens) + graph += pynini.closure(insert_space + graph) + + else: + + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.tens + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.tens + + pynini.closure(insert_space + cardinal.tens, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,010" + ) + + # Technically decimals should be space delineated groups of three, e.g. (1,333 333). This removes any possible spaces + strip_formatting = pynini.cdrewrite(delete_space, "", "", NEMO_SIGMA) + graph = strip_formatting @ graph + + self.graph = graph.optimize() + + graph_separator = pynutil.delete(decimal_separator) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph_fractional = pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"') + + graph_integer = ( + strip_cardinal_apocope(cardinal.graph) + if deterministic + else pynini.union(cardinal.graph, strip_cardinal_apocope(cardinal.graph)) + ) + + self.graph_integer = pynutil.insert('integer_part: "') + graph_integer + pynutil.insert('"') + final_graph_wo_sign = self.graph_integer + graph_separator + insert_space + self.graph_fractional + + self.final_graph_wo_negative = ( + final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal.graph).optimize() + ) + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..a57f1c30c7da7b08d30882c5633f72bbe6641f35 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py @@ -0,0 +1,107 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + period, + protocol_string, + username_string, + www, +) +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + +# common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc.def@studente.università.it" -> electronic { username: "abc.def" domain: "studente.università.it" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + e.g. "https://github.com/NVIDIA/electronic.py" -> electronic { protocol: "https://" domain: "github.com/NVIDIA/electronic.py" preserve_order: true } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_period = pynini.difference(symbols, dot) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + # domains + domain = dot + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) + + # email + username = ( + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) + ) + email = username + domain_graph + + # social media tags + tag = ( + pynutil.delete(at) + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) + ) + + # url + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) + protocol_end = ( + pynini.accep(www + period) + if deterministic + else pynini.accep(www + period) | pynini.cross((www + period), "vu vu vu.") + ) + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..9ecd3c81b9eab2f0777b27c5c0a4db2c3dabc375 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py @@ -0,0 +1,173 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) +suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv")) + + +def singular_to_plural(): + _o = NEMO_SIGMA + pynini.cross("o", "") + pynutil.insert("i") + _a = NEMO_SIGMA + pynini.cross("a", "") + pynutil.insert("e") + _e = NEMO_SIGMA + pynini.cross("e", "") + pynutil.insert("i") + + graph_plural = plurals._priority_union(suppletive, pynini.union(_o, _a, _e), NEMO_SIGMA).optimize() + + return graph_plural + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "2,4 g" -> measure { cardinal { integer_part: "due" fractional_part: "quattro" units: "grammi" preserve_order: true } } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(suppletive) + graph_unit_plural |= graph_unit_singular @ pynini.cdrewrite(convert_space(suppletive), "", "[EOS]", NEMO_SIGMA) + optional_graph_negative = pynini.closure("-", 0, 1) + + graph_unit_denominator = ( + pynini.cross("/", "per") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + + (graph_unit_plural + (optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + unit_singular_graph = ( + pynutil.insert("units: \"") + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + + subgraph_cardinal = ( + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_plural + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) + @ cardinal.fst + @ pynini.cdrewrite(pynini.cross("uno", "un"), "", "", NEMO_SIGMA) + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_singular_graph + ) + + cardinal_dash_alpha = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" }") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + cardinal_times = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | cardinal_times + ) + # final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..ef1f66e1cbbe05fe016e8e0f6246cf49eb1d2948 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/money.py @@ -0,0 +1,176 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import decimal_separator +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + +maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) +fem_plural = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc_plural = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "€1" -> money { currency_maj: "euro" integer_part: "un"} + "€1,000" -> money { currency_maj: "euro" integer_part: "un" } + "4,2 £" -> money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + maj_singular_graph = maj_singular + min_singular_graph = min_singular + maj_plural_graph = maj_singular @ (fem_plural | masc_plural) + min_plural_graph = min_singular @ (fem_plural | masc_plural) + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_singular_graph + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_plural_graph + pynutil.insert("\"") + + graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "un") + pynutil.insert("\"") + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, # 1,05 $ + ) + graph_decimal_plural = ((NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_plural + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, # 1,05 $ + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") + ) + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert("currency_min: \"") + min_singular_graph + pynutil.insert("\"") + graph_min_plural = pynutil.insert("currency_min: \"") + min_plural_graph + pynutil.insert("\"") + + # format ** euro ** centesimo + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + # non zero integer part + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ pynini.cross("1", "un") + + pynutil.insert("\"") + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_no_zero + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.two_digit_no_zero + + pynutil.insert("\""), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..dc90612cb6d58a3f6e4fc83b5e07672ce88e4de0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/time.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 15:30:30 tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + 12:15 tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + 03:38 tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + labels_hour = [str(x) for x in range(0, 25)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + graph_hour = pynini.union(*labels_hour) + + graph_minute_single = pynini.union(*labels_minute_single) + graph_minute_double = pynini.union(*labels_minute_double) + + final_graph_hour_only = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_hour = ( + pynutil.insert("hours: \"") + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert("\"") + ) + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + + graph_hm = ( + final_graph_hour + pynutil.delete(":") + (pynutil.delete("00") | (insert_space + final_graph_minute)) + ) + + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"0\"") | (insert_space + final_graph_minute)) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"0\"") | (insert_space + final_graph_second)) + + pynutil.insert(" preserve_order: true") + ) + + graph_h = final_graph_hour_only + + final_graph = (graph_hm | graph_h | graph_hms).optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..5a57109ac9b02855d829a3600e1fe8ca60a1e4f0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.decimals import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State aRchive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_it_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + word_graph = WordFst(deterministic=deterministic).fst + + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + + self.electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = self.electronic.fst + + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + measure_graph = self.measure.fst + + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + money_graph = self.money.fst + + self.time = TimeFst(deterministic=deterministic) + time_graph = self.time.fst + + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 0.0) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..1d1e90dc12811ba0bbfc12d44f9e30c9661c3e2d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "dr." -> tokens { name: "dottor" } + "Avv." -> tokens { name: "avvocato" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..944e1bbb3fe6024b78d15192431b226af5faaca6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. mangiare -> tokens { name: "mangiare" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..be8bdb5addc0c542fe49d252e1a4240c5d61037f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..af4a003ac2e9a049d473662a42b55bb580033e39 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "due" } -> "due" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "meno "), 0, 1) + self.optional_sign = optional_sign + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + self.numbers = integer + graph = optional_sign + self.numbers + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc6f080e7effda13ee8aefcb571c37fca92d1e8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "venti" fractional_part: "trentaquattro" quantity: "miliardi" } -> + meno venti virgola trentaquattro + decimal { integer_part: "un milione" fractional_part: "zero zero zero" quantity: "milioni" preserve_order: true } --> + un milione virgola zero zero zero + decimal { integer_part: "due" quantity: "milioni" preserve_order: true } --> + due milioni + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "meno ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" virgola ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + self.graph = (graph + delete_preserve_order).optimize() + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..f74f7a053240bf1be7286d93c0d513df2bce1b1c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py @@ -0,0 +1,88 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( # Common string literals; expand as you see fit. + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + colon, + delete_preserve_order, + domain_string, + double_quotes, + protocol_string, + username_string, +) +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc.def2" domain: "studenti.università.it" } -> + "a b c punto d e f due chiocciola s t u d e n t i punto u n i v e r s i t à punto IT + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + graph_digit = digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) + user_name @= verbalize_characters + + convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common | domain_common + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) + protocol = ( + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete(double_quotes) + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("chiocciola ") + domain + | (pynutil.insert("chiocciola ") + user_name) + ) + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..123835c522f5d923df04cf8af586385928293c68 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "due" units: "grammi" } } -> "due grammi" + + Args: + decimal: decimal GraphFst + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = decimal.fst + graph_cardinal = cardinal.fst + + graph = (graph_cardinal | graph_decimal) + pynini.accep(" ") + unit + + graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..e5c340796a9f6dd44aa83ab3c1df5147034a1672 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py @@ -0,0 +1,160 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos2.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + +fem_singular = pynini.project(fem, "input") +masc_singular = pynini.project(masc, "input") + +fem_plural = pynini.project(fem, "output") +masc_plural = pynini.project(masc, "output") + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "euro" integer_part: "un"} -> "un euro" + money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } -> "quattro sterline venti penny" + + Args: + decimal: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + maj_singular_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + maj_singular_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + maj_plural_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + maj_plural_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + maj_masc = maj_plural_masc | maj_singular_masc + maj_fem = maj_plural_fem | maj_singular_fem + + min_singular_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + min_singular_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + min_plural_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + min_plural_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + min_masc = min_plural_masc | min_singular_masc + min_fem = min_plural_fem | min_singular_fem + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1) + + graph_integer_masc = integer_part + NEMO_SPACE + maj_masc + graph_integer_fem = shift_cardinal_gender(integer_part) + NEMO_SPACE + maj_fem + + graph_integer = graph_integer_fem | graph_integer_masc + + graph_integer_with_minor_masc = ( + graph_integer_masc + + NEMO_SPACE + + pynini.union( + optional_add_and + strip_cardinal_apocope(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) + + delete_preserve_order + ) + + graph_integer_with_minor_fem = ( + graph_integer_fem + + NEMO_SPACE + + pynini.union( + optional_add_and + shift_cardinal_gender(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) + + delete_preserve_order + ) + + graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc + + graph_decimal_masc = decimal.graph + NEMO_SPACE + maj_masc + + graph_decimal_fem = decimal.graph + graph_decimal_fem += NEMO_SPACE + maj_fem + + graph_decimal = graph_decimal_fem | graph_decimal_masc + graph_decimal = ( + pynini.cdrewrite( + pynutil.insert(" di"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph_decimal + ) + + graph_minor_masc = fractional_part + NEMO_SPACE + min_masc + delete_preserve_order + graph_minor_fem = shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem + delete_preserve_order + + graph_minor = graph_minor_fem | graph_minor_masc + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..adfe70afa88f69f51b0924aab5da2c979894680e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_preserve_order +from zonos2.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + Args: + cardinal_tagger: cardinal_tagger tagger GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + number_verbalization = graph_zero | cardinal_tagger.two_digit_no_zero + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + hour_verbalized = hour @ number_verbalization + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + + graph_hms_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hms_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hm_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + ) + + graph_hm_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + ) + + graph_h = hour_verbalized + + self.graph = graph_hms_30 | graph_hms_30 | graph_hms_15 | graph_hm_30 | graph_hm_15 | graph_h + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..d383f6646ec3f619e4f26190497f278bdf415ce0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst as CardinalTagger +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) + measure_graph = measure.fst + money = MoneyFst(decimal=decimal, deterministic=deterministic) + money_graph = money.fst + cardinal_tagger = CardinalTagger(deterministic=deterministic) + time = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic) + time_graph = time.fst + + graph = ( + cardinal_graph + | decimal_graph + | electronic_graph + | whitelist_graph + | measure_graph + | money_graph + | time_graph + ) + + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..bbaf255ff0b227c93c2154b8cb9eeaf6150d7478 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"it_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ef03d642b135692333ff4a51032d9d7726a1f4b7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv new file mode 100644 index 0000000000000000000000000000000000000000..09258cb4c8d742834f6d7994580e16d863fe2783 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv @@ -0,0 +1,31 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b932b7fb3fae11c5ef32e39ed90739d3b136ff86 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv @@ -0,0 +1,12 @@ +令和 +平成 +昭和 +大正 +明治 +西暦 +和暦 +西洋暦 +グレゴリオ暦 +紀元前 +紀元 +紀元後 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1c95675c27ddf66508601e68834ccf48e463a3f9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv @@ -0,0 +1,5 @@ +R. 令和 +H. 平成 +S. 昭和 +T. 大正 +M. 明治 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f992b4d28d2509cd2b4822ac28690259d64917f8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv @@ -0,0 +1,12 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv new file mode 100644 index 0000000000000000000000000000000000000000..41556b4db2f74a429f6e571e46450f57dddbb868 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv @@ -0,0 +1,15 @@ +月 月曜日 +火 火曜日 +水 水曜日 +木 木曜日 +金 金曜日 +土 土曜日 +日 日曜日 +祝 祝日 +月曜日 +火曜日 +水曜日 +木曜日 +金曜日 +土曜日 +日曜日 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ede6b97b73ae912917667f3a00a9efb71580bee3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1585fe836701348ae4543e3001c2a10151d750f2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..562e632656a6b6fd752296d2a30e419206052bf5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +2 二十 +3 三十 +4 四十 +5 五十 +6 六十 +7 七十 +8 八十 +9 九十 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7fab21718dad9ed8089512e9d513cab84074dfc2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 零 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..67ad785f0e2e9e4eec5c13063a884f10e8904160 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv @@ -0,0 +1,23 @@ +& アンド +# ハッシュタグ +@ アット +§ セクション +™ トレードマーク +® 登録商標マーク +© 著作権 +_ アンダースコア +% パーセント +* 星印 ++ プラス +/ スラッシュ += エコール +^ 曲折アクセント記号 +| 縦棒 +~ ティルダ +$ ドール +£ ポンド +€ ユーロ +₩ ウォン +¥ 円 +° 度 +º 度 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f6beedce80aec13a68e584811154d73bd47609e1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv @@ -0,0 +1,23 @@ +今朝 +今夜 +今晩 +午前 +午後 +夕方 +夜中 +夜 +夜半 +朝 +早朝 +明け方 +昼 +深夜 +毎朝 +毎夜 +毎晩 +毎日 +真夜中 +翌日 +未明 +正午 +真夜中の \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1d5c084253396e5d4183fa198f14241fcc175a95 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv @@ -0,0 +1,24 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5e8276a00502b43948252b6a5af495e33d580e2f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5e8276a00502b43948252b6a5af495e33d580e2f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d0d7bef709e55ad4823f9b8b8d9ef9ababbc610d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv @@ -0,0 +1,40 @@ +Dr. ドクター +dr. ドクター +Mr. ミスター +mr. ミスター +Ms. ミス +ms. ミス +Mrs. ミシーズ +mrs. ミシーず +st. ストリード +St. ストリード +mt. マウント +Mt. マウント +Prof. プロフェッサー +prof. プロフェッサー +sr. シニア +Sr. シニア +jr. ジュニア +Jr. ジュニア +rd. ロード +Rd. ロード +Ave. アベニュー +ave. アベニュー +no. ナンバー +No. ナンバー +( カッコ +) カッコ閉じる ++ プラス +- マイナス +Σ シグマ +η エタ +κ カッパ +ω オメガ +σ シグマ +α アルフ +ν ヌー +δ デルタ +ι イオタ +vs. バーサス +Ph.D. p h d +etc. エトセトラ diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7ab7a2c4ed0620c01d209b4ac1eb62ae5e51f71e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py @@ -0,0 +1,297 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_NARROW_NON_BREAK_SPACE = u"\u202f" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() +NEMO_SPACES_AND_ALHPANUMERICS = pynini.closure( + NEMO_SIGMA + | pynini.closure(NEMO_SPACE) + | pynini.closure(NEMO_NON_BREAKING_SPACE) + | pynini.closure(NEMO_DIGIT) + | pynini.closure(NEMO_WHITE_SPACE) +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..1a4121f70eabe4ada6f2d6d3ac36da98665342e8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py @@ -0,0 +1,191 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "二十三" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + + no_one = pynini.difference(NEMO_DIGIT, "1") + no_zero_and_one = pynini.difference(no_one, "0") + + graph_digit_alt = no_zero_and_one @ graph_digit + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + + graph_all = (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = (pynini.cross('1', '百') | (graph_digit_alt + pynutil.insert('百'))) + pynini.union( + pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_all) + ) + graph_hundred = hundreds @ graph_hundred_component + + thousands = NEMO_DIGIT**4 + graph_thousand_component = (pynini.cross('1', '千') | (graph_digit_alt + pynutil.insert('千'))) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_component_alt = (graph_digit + pynutil.insert('千')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + # this grammar is for larger number in later gramamr + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = (graph_digit + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + self.man = graph_ten_thousand.optimize() + + hundred_thousands = NEMO_DIGIT**6 + hundred_thousands_position = NEMO_DIGIT**2 + hundred_thousands_position = hundred_thousands_position @ graph_all + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + million_position = NEMO_DIGIT**3 + million_position = million_position @ graph_hundred_component + graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + ten_million_position = NEMO_DIGIT**4 + ten_million_position = ten_million_position @ graph_thousand_component_alt + graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + thousand_millions_position = NEMO_DIGIT**2 + thousand_millions_position = thousand_millions_position @ graph_all + graph_thousand_million_component = (thousand_millions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + ten_billions = NEMO_DIGIT**11 + ten_billions_position = NEMO_DIGIT**3 + ten_billions_position = ten_billions_position @ graph_hundred_component + graph_ten_billions_component = (ten_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**12 + hundred_billions_position = NEMO_DIGIT**4 + hundred_billions_position = hundred_billions_position @ graph_thousand_component_alt + graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + graph = pynini.union( + graph_hundred_billions, + graph_ten_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_all, + graph_zero, + ) + self.just_cardinals = graph.optimize() + + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + ) + + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..54cc52d45ff81eeb2c32a4f623b2b80e127889c1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py @@ -0,0 +1,227 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NON_BREAKING_SPACE, + GraphFst, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 2024/01/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024/1/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024-01-30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024.01.30 -> date { year: "二千二十四" month: "一" day: "三十" } + H.6 -> date { era: "平成" yera: "六年" } + R.1 -> date { era: "令和" "year: "元年" } + S.5 -> date { era: "昭和" "year: "五年" } + T.5 -> date { era: "大正" "year: "五年" } + M.5 -> date { era: "明治" "year: "五年" } + 21日月曜日 -> tokens { date { day: "二十一日" weekday: "月曜日" } } + 70年代 -> date { year: "七十年代" } + 西暦794年 -> tokens { date { era: "西暦" year: "七百九十四年" } } + 1月1日(月)〜3日(水) + -> tokens { date { month: "一月" day: "一日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { day: "三日" weekday: "水曜日" } } + 70〜80年代 + -> tokens { cardinal { integer: "七十" } } tokens { name: "から" } tokens { date { year: "八十年代" } } + 7月5〜9日(月〜金) + -> tokens { date { month: "七月" } } tokens { cardinal { integer: "五" } } tokens { name: "から" } tokens { date { day: "九日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { weekday: "金曜日" } } + 7月初旬〜9月中旬 + -> tokens { date { month: "七月" } } tokens { name: "初" } tokens { name: "旬" } tokens { name: "から" } tokens { date { month: "九月" } } tokens { name: "中" } tokens { name: "旬" } + 3〜4月 + -> tokens { cardinal { integer: "三" } } tokens { name: "から" } tokens { date { month: "四月" } } + 2023年3月1日(水)〜6月12日(火) + -> tokens { date { year: "二千二十三年" month: "三月" day: "一日" weekday: "水曜日" } } tokens { name: "から" } tokens { date { month: "六月" day: "十二日" weekday: "火曜日" } } + 10月中旬〜11月上旬 + -> tokens { date { month: "十月" } } tokens { date { month: "中旬" } } tokens { name: "から" } tokens { date { month: "十一月" } } tokens { date { month: "上旬" } } + 1976年7月17日〜8月1日 + -> tokens { date { year: "千九百七十六年" month: "七月" day: "十七日" } } tokens { name: "から" } tokens { date { month: "八月" day: "一日" } } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + + month = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/month.tsv")) + day = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/day.tsv")) + week = pynini.string_file(get_abs_path("data/date/week.tsv")) + era = pynini.string_file(get_abs_path("data/date/era.tsv")) + era_abbrev = pynini.string_file(get_abs_path("data/date/era_abbrev.tsv")) + + signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + delete_spaces = pynini.closure( + pynutil.delete(" ") | pynutil.delete(NEMO_NARROW_NON_BREAK_SPACE) | pynutil.delete(NEMO_NON_BREAKING_SPACE) + ) + + era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") + era_abbrev_component = pynutil.insert("era: \"") + era_abbrev + pynutil.insert("\"") + year_component = pynutil.insert("year: \"") + graph_cardinal + pynutil.insert("年") + pynutil.insert("\"") + month_component = pynutil.insert("month: \"") + month + pynutil.insert("月") + pynutil.insert("\"") + day_component = pynutil.insert("day: \"") + day + pynutil.insert("日") + pynutil.insert("\"") + + front_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + preceding_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + # this graph optionally accepts () around weekday to accomodate to inputs like (月〜金), thus being longer + + week_component = ( + (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynini.cross("〜", "から") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynutil.delete("・") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + ) + + # era, year, month, date + graph_basic_date = ( # (R.|令和)2024/01/01, (R.|令和)2024/01/01/(水), (R.|令和)01/01, (R.|令和)01/01(水) + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynini.closure(year_component + signs + pynutil.insert(" "), 1) + + month_component + + signs + + pynutil.insert(" ") + + day_component + + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + ) + + # 2024年, 9月, 28日 + individual_year_component = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + pynini.accep("年") + + pynutil.insert("\"") + ) + # this extra individual year component is to accomodate inputs R. 2024 with out "年" + # the inputs may or maynot include "年", thus below: + individual_year_component_2 = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + (pynini.accep("世紀") | pynini.accep("")) + + pynutil.insert("\"") + ) | ( + era_abbrev_component + + pynutil.insert(" ") + + pynutil.insert("year: \"") + + graph_cardinal + + pynutil.insert("年") + + pynutil.insert("\"") + ) + + individual_month_component = ( + pynutil.insert("month: \"") + month + pynini.accep("月") + pynutil.insert("\"") + ) | ( + pynutil.insert("month: \"") + + (pynini.accep("中旬") | pynini.accep("下旬") | pynini.accep("上旬")) + + pynutil.insert("\"") + ) + individual_day_component = ( + pynutil.insert("day: \"") + graph_cardinal + pynini.accep("日") + pynutil.insert("\"") + ) + + graph_individual_component = ( + individual_year_component + | individual_month_component + | individual_day_component + | week_component + | individual_year_component_2 + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + # combined the above individual date components + graph_individual_component_combined = ( + (individual_year_component + pynutil.insert(" ") + individual_month_component) + | (individual_month_component + pynutil.insert(" ") + individual_day_component) + | ( + individual_year_component + + pynutil.insert(" ") + + individual_month_component + + pynutil.insert(" ") + + individual_day_component + ) + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + nendai = pynini.accep("年代") + era_nendai = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + nendai + + pynutil.insert("\"") + ) + + graph_all_date = ( + graph_basic_date | graph_individual_component | graph_individual_component_combined | era_nendai + ) + + final_graph = graph_all_date + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..413322570c946f8a39ed9ef9f19b6f9edecc8b46 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 0.5 -> decimal { integer_part: "零" fractional_part: "五" } + -0.5万 -> decimal { negative: "マイナス" integer_part: "零" fractional_part: "五" quantity: "万"} + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_before_decimal = cardinal.just_cardinals + cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") + graph_fraction = ( + pynutil.insert("fractional_part: \"") + + pynini.closure((cardinal_after_decimal | zero), 1) + + pynutil.insert("\"") + ) + graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction + + graph_optional_sign = ( + pynutil.insert("negative: \"") + + (pynini.cross("-", "マイナス") | pynini.accep("マイナス")) + + pynutil.insert("\"") + ) + + graph_decimal = graph_decimal_no_sign | (graph_optional_sign + pynutil.insert(" ") + graph_decimal_no_sign) + + self.just_decimal = graph_decimal_no_sign.optimize() + + final_graph = self.add_tokens(graph_decimal) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..2fa13cc6ce144ba9db32ff5746fac8ece964caa5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/fraction.py @@ -0,0 +1,125 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fractions, e.g. + 1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } + 1と3/4 -> fraction { integer: "一" denominator: "四" numerator: "三" } + 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } + 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } + マイナス1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal = cardinal.just_cardinals + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + slash = pynutil.delete('/') + morphemes = pynini.accep('分の') + root = pynini.accep('√') + + decimal_number = ( + cardinal + + pynini.cross(".", "点") + + pynini.closure(pynini.closure(graph_digit) | pynini.closure(graph_zero)) + ) + + integer_component = ( + pynutil.insert('integer_part: \"') + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + integer_component_with_char = ( + pynutil.insert('integer_part: \"') + + ( + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + (pynini.accep("と") | pynini.accep("荷")) + ) + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + denominator_component = ( + pynutil.insert("denominator: \"") + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + numerator_component = ( + pynutil.insert("numerator: \"") + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + + # 3/4, 1 3/4, 1と3/4, -3/4, -1 3/4, 1と3/4, √1と3/4 and any combination of root number, cardinal number and decimal number + graph_fraction_slash = ( + pynini.closure( + (integer_component + pynini.accep(NEMO_SPACE)) + | (integer_component_with_char + pynutil.insert(NEMO_SPACE)), + 0, + 1, + ) + + numerator_component + + slash + + pynutil.insert(NEMO_SPACE) + + denominator_component + ) + + # 4分の3 -4分の3 and any combs + graph_fraction_word = ( + pynini.closure( + ( + integer_component + pynini.accep(NEMO_SPACE) + | integer_component_with_char + pynutil.insert(NEMO_SPACE) + ), + 0, + 1, + ) + + denominator_component + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("morphosyntactic_features: \"") + + morphemes + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + optional_sign = ( + pynutil.insert("negative: \"") + + (pynini.accep("マイナス") | pynini.cross("-", "マイナス")) + + pynutil.insert("\"") + ) + + graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + ( + graph_fraction_slash | graph_fraction_word + ) + + graph = graph_fraction_slash_sigh # | + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..c88b59d404e958d5e515298d33869fb5beaa1ddd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 第100 -> ordinal { integer: "第百" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + morpheme_pre = pynini.accep('第') + morpheme_post = pynini.accep('番目') + graph_ordinal = pynini.union(morpheme_pre + graph_cardinal, graph_cardinal + morpheme_post) + + final_graph = pynutil.insert("integer: \"") + graph_ordinal + pynutil.insert("\"") + + graph_ordinal_final = self.add_tokens(final_graph) + self.fst = graph_ordinal_final.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/punctuation.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..87256d13372bdf106e8dac4031b8df23a67dc913 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/punctuation.py @@ -0,0 +1,70 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#$%&'()*+,-./:;<=>?@^_`{|}。，；：《》“”·~【】！？、‘’.<>-——_、。.「」『』‘`／・；’”“”‷･〔〕々〃ゝゞヽ〲〱〳〴〵ヾ〆，~" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + whitelist_symbols = load_labels(get_abs_path("data/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + range_component = pynini.cross("〜", "から") | pynini.accep( + "から" + ) # forcing this conversion for special tilde + + self.graph = punct | pynutil.add_weight(range_component, -1.0) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..44bbb6403ee3c1220950867952e1c178ce99754a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/time.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 1時30分 -> time { hours: "一" minutes: "三十" } + 今夜0時 -> time { suffix: "今夜" hours: "零" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + + hour_clock = pynini.string_file(get_abs_path("data/time/hour.tsv")) + minute_clock = pynini.string_file(get_abs_path("data/time/minute.tsv")) + second_clock = pynini.string_file(get_abs_path("data/time/second.tsv")) + division = pynini.string_file(get_abs_path("data/time/division.tsv")) + + division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + hour_component = ( + pynutil.insert("hours: \"") + + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + + (pynini.accep("時") | pynini.accep("時間") | pynini.accep("時頃")) + + pynutil.insert("\"") + ) + minute_component = pynutil.insert("minutes: \"") + ( + graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal) + ) + pynini.accep("分") + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) + pynutil.insert( + "\"" + ) | ( + pynutil.insert("minutes: \"") + + pynini.accep("半") + + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) + + pynutil.insert("\"") + ) + second_component = ( + pynutil.insert("seconds: \"") + + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + + pynini.accep("秒") + + pynutil.insert("\"") + ) + + graph_individual_time = pynini.closure(division_component + pynutil.insert(" "), 0, 1) + ( + hour_component + | minute_component + | second_component + | (hour_component + pynutil.insert(" ") + minute_component) + | (hour_component + pynutil.insert(" ") + minute_component + pynutil.insert(" ") + second_component) + | (minute_component + pynutil.insert(" ") + second_component) + ) + + colon = pynutil.delete(":") + hour_clock_component = ( + pynutil.insert("hours: \"") + + pynutil.delete("0").ques + + hour_clock + + pynutil.insert("時") + + pynutil.insert("\"") + ) + minute_clock_component = ( + pynutil.insert("minutes: \"") + + pynutil.delete("0").ques + + minute_clock + + pynutil.insert("分") + + pynutil.insert("\"") + ) + second_clock_component = ( + pynutil.insert("seconds: \"") + + pynutil.delete("0").ques + + second_clock + + pynutil.insert("秒") + + pynutil.insert("\"") + ) + + graph_clock = ( + hour_clock_component + + pynutil.insert(" ") + + colon + + minute_clock_component + + pynutil.insert(" ") + + colon + + second_clock_component + ) | (hour_clock_component + pynutil.insert(" ") + colon + minute_clock_component) + + graph = graph_individual_time | graph_clock + + graph_final = self.add_tokens(graph) + self.fst = graph_final.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..fb0f5b8d3824d92f4275b3532d7f91d33bedea8f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py @@ -0,0 +1,94 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, generator_main +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + else: + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(cardinal=cardinal, deterministic=deterministic) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(cardinal=cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(word.fst, 100), + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") + tagger = pynini.closure(token, 1) + + self.fst = tagger + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..d0a58d7535d49a72d50102d6af1f1824bc799dbd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. Dr. -> tokens { name: "ドクター" } + Ms. -> { name: "ミス" } + jr. -> { name: "ジュニア" } + etc. -> { name: "エトセトラ" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..e8661ace46b94c814b416a9e1151505021a92155 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. 文字 -> tokens { name: "文字" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..65523afed35dba7786d4fdbca6b49e43abfc648e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/utils.py @@ -0,0 +1,56 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +# def load_labels(abs_path): +# """ +# loads relative path file as dictionary + +# Args: +# abs_path: absolute path + + +# Returns dictionary of mappings +# """ +# #label_tsv = open(abs_path, encoding="utf-8") +# label_tsv = open(abs_path, "r") +# labels = list(csv.reader(label_tsv, delimiter="\t")) +# return labels +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..44016bc73489c742c6ace486c8aa6d9c4d066a15 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { integer: "一" } -> 一 + cardinal { negative: "-" integer: "二十三" } -> マイナス二十三 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + optional_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = pynini.closure(optional_sign, 0, 1) + graph + + final_graph = self.delete_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..5d172e579ce98c092097f09f45f382e07005d41a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/date.py @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date e.g. + date { year: "二千二十四" month: "三" day: "四" } -> 二千二十四年三月四日 + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + era_component = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + year_component = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + month_component = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + day_component = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + week_component = ( + pynutil.delete("weekday: \"") + + pynini.closure(delete_space) + + pynini.closure(NEMO_NOT_QUOTE) + + pynini.closure(delete_space) + + pynutil.delete("\"") + ) + + graph_basic_date = ( + pynini.closure(era_component + delete_space, 0, 1) + + pynini.closure(year_component + delete_space, 0, 1) + + pynini.closure(month_component + delete_space, 0, 1) + + pynini.closure(day_component, 0, 1) + + pynini.closure((delete_space + week_component) | (week_component), 0, 1) + ) | month_component + delete_space + week_component + + final_graph = graph_basic_date + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..38549132960d5b0ef6ae759438b632b6beaeaa25 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { integer_part: "零" fractional_part: "五" } -> 零点五 + decimal { positive: "-" integer_part: "零" fractional_part: "五" } -> マイナス零点五 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + graph_integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph_fraction = ( + pynutil.delete("fractional_part: \"") + + pynutil.insert("点") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_decimal_no_sign = graph_integer + pynutil.delete(" ") + graph_fraction + graph_decimal = graph_decimal_no_sign | (graph_optional_sign + pynutil.delete(" ") + graph_decimal_no_sign) + + delete_tokens = self.delete_tokens(graph_decimal) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..61d71df9a463c75aa4948ebb56b526a4d0b61a0f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py @@ -0,0 +1,123 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fractionss, e.g. + tokens { fraction { denominator: "二" numerator: "一"} } -> 1/2 + tokens { fraction { integer: "一" denominator: "四" numerator: "三" } } -> 1と3/4 + tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一荷四分の三 + tokens { fraction { denominator: "√3" numerator: "1" } } -> ルート三分の一 + tokens { fraction { denominator: "1.65" numerator: "50" } } -> 一点六五分の五十 + tokens { fraction { denominator: "二" numerator: "一"} } -> マイナス1/2 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + denominator_component = ( + pynutil.delete('denominator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + ) + numerator_component = ( + pynutil.delete('numerator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + ) + + # 1/3 + graph_regular_fraction = ( + denominator_component + pynutil.delete(NEMO_SPACE) + pynutil.insert("分の") + numerator_component + ) + + denominator_component_root = ( + pynutil.delete('denominator: \"') + + pynini.cross("√", "ルート") + + pynini.closure(NEMO_NOT_QUOTE - "√") + + pynutil.delete("\"") + ) + numerator_component_root = ( + pynutil.delete('numerator: \"') + + pynini.cross("√", "ルート") + + pynini.closure(NEMO_NOT_QUOTE - "√") + + pynutil.delete("\"") + ) + # √3/1 + graph_regular_fraction_root = ( + (denominator_component_root | denominator_component) + + pynutil.delete(NEMO_SPACE) + + pynutil.insert("分の") + + (numerator_component_root | numerator_component) + ) + + # 3分の1 + graph_regular_fraction_char = ( + (denominator_component | denominator_component_root) + + pynutil.delete(NEMO_SPACE) + + pynutil.delete("morphosyntactic_features: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + pynutil.delete(NEMO_SPACE) + + (numerator_component | numerator_component_root) + ) + + graph_integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(pynini.cross("√", "ルート"), 0, 1) + + pynini.closure( + NEMO_NOT_QUOTE - pynini.union("荷", "と", "√") + ) # had to remove these 3 items fron nemo_not _quote so the root is properly converted in a deterministic way. + + pynutil.insert("荷") + + pynutil.delete("\"") + ) + + graph_integer_with_char = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(pynini.cross("√", "ルート"), 0, 1) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union("荷", "と", "√")) + + (pynini.accep("と") | pynini.accep("荷")) + + pynutil.delete("\"") + ) + + graph_regular_integer = ( + (graph_integer | graph_integer_with_char) + + delete_space + + (graph_regular_fraction | graph_regular_fraction_root | graph_regular_fraction_char) + ) + + optional_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + + graph = pynini.closure(optional_sign, 0, 1) + ( + graph_regular_integer | graph_regular_fraction | graph_regular_fraction_root | graph_regular_fraction_char + ) + + # graph = pynini.closure(graph_optional_sign, 0, 1) + graph_fractions + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..880227482052476221819752071a49369b6e7393 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal e.g. + tokens { ordinal { integer: "第一千万" } } -> 第一千万 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_integer = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = graph_integer + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..51f3e8737fa81cccc66212027acf5e5e31a1e5cf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_SPACE, + NEMO_SIGMA, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + + remove_space_around_single_quote = pynini.cdrewrite( + delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + ) + # this works if spaces in between (good) + # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA + + graph = remove_space_around_single_quote.optimize() + + return graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..5507ea5d0c12e7b3c79dbba470d9db0242b6e410 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py @@ -0,0 +1,81 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_PUNCT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class PostProcessor(GraphFst): + ''' + Postprocessing of TN, now contains: + 1. punctuation removal + 2. letter case conversion + 3. oov tagger + ''' + + def __init__( + self, + remove_puncts: bool = False, + to_upper: bool = False, + to_lower: bool = False, + tag_oov: bool = False, + ): + super().__init__(name="PostProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if remove_puncts: + remove_puncts_graph = pynutil.delete( + pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) + ) + graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() + + if to_upper or to_lower: + if to_upper: + conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) + else: + conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) + + graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() + + if tag_oov: + zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) + zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) + + zh_charset = ( + zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) + ) + en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE + charset = zh_charset | en_charset + + with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: + tags = f.readline().strip().split('\t') + assert len(tags) == 2 + ltag, rtag = tags + + oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) + tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) + graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..bd7daa34f2e458faab602106bae1708d613db630 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/time.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time e.g. + + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_component = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + second_component = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + division_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_basic_time = pynini.closure(division_component + pynutil.delete(" "), 0, 1) + ( + (hour_component + pynutil.delete(" ") + minute_component + pynutil.delete(" ") + second_component) + | (hour_component + pynutil.delete(" ") + minute_component) + | hour_component + | minute_component + | second_component + ) + + final_graph = graph_basic_time + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..5c51d935f4355e1890ccfe39bf247cad9b590931 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py @@ -0,0 +1,66 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + date = DateFst(deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + + # money = MoneyFst(decimal=decimal, deterministic=deterministic) + # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + + graph = pynini.union( + date.fst, + cardinal.fst, + ordinal.fst, + decimal.fst, + fraction.fst, + word.fst, + time.fst, + whitelist.fst, + ) + graph = pynini.closure(delete_space) + graph + pynini.closure(delete_space) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..edbc8754fd009dc292d0a853487533091810faa1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.postprocessor import PostProcessor +from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst + +# from nemo.utils import logging + + +class VerbalizeFinalFst(GraphFst): + """ """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"jp_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) + + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) + + self.fst = (verbalizer @ postprocessor.fst).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..f44ea414e0ddd028b4ac549e1a73d0b4a1a8b227 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "ミスター" } } -> ミスター + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..a1452847c2120826ecf0c63043b2791fcaf41737 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ja/verbalizers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + ''' + tokens { char: "文字" } -> 文字 + ''' + + def __init__(self, deterministic: bool = True): + super().__init__(name="char", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..725f61402d7be057471db8a757189f83533fbaba --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2f54cee9202f9560342238f8b3fa87c62c0d40ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv @@ -0,0 +1,2 @@ +6 유 +10 시 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/week.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/week.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bc205bc3fc62050b5456aa2b4452e805c5bf32d9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/date/week.tsv @@ -0,0 +1,8 @@ +월 월요일 +화 화요일 +수 수요일 +목 목요일 +금 금요일 +토 토요일 +일 일요일 +공 공휴일 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f562cfbad85a2b894e34df79a89fdd9f19361f14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv @@ -0,0 +1,11 @@ +카드 끝자리 카드 끝자리 +카드 마지막 네자리 카드 마지막 네자리 +카드 마지막 4자리 카드 마지막 네자리 +신용카드 번호 신용카드 번호 +신용카드 신용카드 +체크카드 번호 체크카드 번호 +체크카드 체크카드 +카드번호 카드번호 +결제 카드 결제 카드 +결제카드 결제카드 +카드 카드 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3d04ca29838cc4e2839b37fc02b664cdbf03a695 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv @@ -0,0 +1,28 @@ +.com 닷컴 +.org 닷 오알지 +.gov 닷 거브 +.edu 닷 에듀 +.net 닷 넷 +.ai 닷 에이아이 +.io 닷 아이오 +.dev 닷 데브 +.app 닷 앱 +.cloud 닷 클라우드 +.shop 닷 샵 +.store 닷 스토어 +.co 닷 씨오 +.me 닷 미 +.kr 닷 케이알 +.co.kr 닷 씨오 닷 케이알 +.ac.kr 닷 에이씨 닷 케이알 +.or.kr 닷 오알 닷 케이알 +.go.kr 닷 지오 닷 케이알 +.re.kr 닷 알이 닷 케이알 +.cn 닷 씨엔 +.fr 닷 에프알 +.de 닷 디이 +.it 닷 아이티 +.uk 닷 유케이 +.br 닷 비알 +.in 닷 아이엔 +.ru 닷 알유 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c80d08a6917ec3e04d228fe1a132f36f63eade9e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv @@ -0,0 +1,6 @@ +.jpg 닷 제이피지 +.png 닷 피엔지 +.pdf 닷 피디에프 +.JPG 닷 제이피지 +.PNG 닷 피엔지 +.PDF 닷 피디에프 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f551dabf40174eed9e239d55b438b3b8809a931d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv @@ -0,0 +1,12 @@ +@ 골뱅이 +. 점 +/ 슬래시 +- 대시 +_ 언더바 +: 콜론 +? 물음표 += 이퀄 +& 앰퍼샌드 +% 퍼센트 ++ 플러스 +# 샵 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ccec41e7f777a5656684203ff909d1aa469def9a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv @@ -0,0 +1,18 @@ +kg 킬로그램 +g 그램 +km 킬로미터 +m 미터 +cm 센티미터 +mm 밀리미터 +L 리터 +l 리터 +mL 밀리리터 +ml 밀리리터 +h 시간 +s 초 +N 뉴턴 +W 와트 +Hz 헤르츠 +° 도 +% 퍼센트 +rpm 분당회전수 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a5ddfeb45ca77d01a65e64473d4abfc67925a0c8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv @@ -0,0 +1,22 @@ +₩ 원 +KRW 원 +krw 원 +$ 달러 +US$ 달러 +HK$ 홍콩 달러 +hk$ 홍콩 달러 +€ 유로 +EUR 유로 +¥ 엔 +JPY 엔 +CAD 캐나다 달러 +cad 캐나다 달러 +NZD 뉴질랜드 달러 +nzd 뉴질랜드 달러 +CHF 스위스 프랑 +chf 스위스 프랑 +AED 아랍에미리트 디르함 +aed 아랍에미리트 디르함 +Dh 디르함 +DH 디르함 +Dhs. 디르함 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6c48f5e7d3418d91eafa51a584558dac4596f356 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv @@ -0,0 +1,16 @@ +개 +명 +병 +마리 +송이 +포기 +사람 +자루 +채 +켤레 +그루 +벌 +잔 +장 +권 +살 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..61a7dddcf7a4d8f0b10a4018ced520f27452061e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/digit.tsv @@ -0,0 +1,9 @@ +1 일 +2 이 +3 삼 +4 사 +5 오 +6 육 +7 칠 +8 팔 +9 구 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f9926b64bae62f0694c282f6bda3e9b3cd58a51e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv @@ -0,0 +1,9 @@ +1 한 +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/tens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d8b8e0a2b6a2c22f410818c8b384466a82a41355 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/tens.tsv @@ -0,0 +1,9 @@ +1 십 +2 이십 +3 삼십 +4 사십 +5 오십 +6 육십 +7 칠십 +8 팔십 +9 구십 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7024c053491118d051d72dc10ea82c0d668bbcbb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/number/zero.tsv @@ -0,0 +1 @@ +0 영 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b3efc4cef2fd05cb1fdbec0b4deb79b7e1030b9d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv @@ -0,0 +1,8 @@ +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ad796a0e8a665ecafc8c1db1a1af8674b316e101 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv @@ -0,0 +1,5 @@ +1 첫 +11 열한 +20 스무 +21 스물한 +31 서른한 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..994a21a1d2d100a5c19cc47b37566b2dfed1494e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv @@ -0,0 +1,2 @@ +10 열 +30 서른 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3111a82dc47762414092c0af425476b702d6b26b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv @@ -0,0 +1,3 @@ +1 열 +2 스물 +3 서른 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/division.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/division.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9250d0a8fba97f883a44552558130c7cabc6ce16 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/division.tsv @@ -0,0 +1,7 @@ +오전 +오후 +새벽 +아침 +낮 +저녁 +밤 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/hour.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/hour.tsv new file mode 100644 index 0000000000000000000000000000000000000000..abfccd31040ae46db799513226ad11c70b32bf5b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/time/hour.tsv @@ -0,0 +1,12 @@ +1 한 +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 +10 열 +11 열한 +12 열두 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..82dc1220e09eba1ee8f7b33c76cdd15f7caab094 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/data/whitelist.tsv @@ -0,0 +1,38 @@ +Dr. 박사 +dr. 박사 +Mr. 씨 +mr. 씨 +Ms. 양 +ms. 양 +Mrs. 여사 +mrs. 여사 +mt. 산 +Mt. 산 +Prof. 교수 +prof. 교수 +sr. 시니어 +Sr. 시니어 +jr. 주니어 +Jr. 주니어 +rd. 로 +Rd. 로 +Ave. 대로 +ave. 대로 +no. 번호 +No. 번호 +( 왼쪽 괄호 +) 오른쪽 괄호 ++ 더하기 +- 마이너스 +Σ 시그마 +η 에타 +κ 카파 +ω 오메가 +σ 시그마 +α 알파 +ν 뉴 +δ 델타 +ι 이오타 +vs. 대 +Ph.D. 박사학위 +etc. 등 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7489143bee63d5d997c684c7684317ec0cbc472f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/graph_utils.py @@ -0,0 +1,173 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import load_labels +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_ALPHA = pynini.union(*[chr(i) for i in range(ord('가'), ord('힣') + 1)]).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, "\u00a0"), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = "lower_cased"): + labels = load_labels(input_file) + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..8b3e5c9a73e5d97de008ab17683dbc1760a7d6b4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/cardinal.py @@ -0,0 +1,327 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Optional small whitespace inside parentheses or after signs + ws = pynini.closure(NEMO_SPACE, 0, 2) + + # Load base .tsv files + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + + digit_except_one = pynini.difference(NEMO_DIGIT, "1") + digit_except_zero_one = pynini.difference(digit_except_one, "0") + + graph_digit_no_zero_one = digit_except_zero_one @ graph_digit + graph_tens = pynini.string_file(get_abs_path("data/number/tens.tsv")) + + # Compose all basic number forms + graph_1_to_99 = (graph_tens + (graph_digit | pynutil.delete('0'))) | graph_digit + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = ( + pynini.cross('1', '백') | (graph_digit_no_zero_one + pynutil.insert('백')) + ) + pynini.union(pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_1_to_99)) + graph_hundred = hundreds @ graph_hundred_component + + thousands = NEMO_DIGIT**4 + graph_thousand_component = pynini.union( + pynini.cross('1', '천'), + graph_digit_no_zero_one + pynutil.insert('천'), + ) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = pynini.union( + pynini.cross('1', '만'), + graph_digit_no_zero_one + pynutil.insert('만'), + ) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + + hundred_thousands = NEMO_DIGIT**6 + graph_hundred_thousand_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert("만")) + pynini.union( + pynini.closure(pynutil.delete("0")), + graph_thousand_component, + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_1_to_99), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + graph_million_component = ((graph_hundred) + pynutil.insert('만')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + graph_ten_million_component = ((graph_thousand) + pynutil.insert('만')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + graph_thousand_million_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + billions = NEMO_DIGIT**11 + graph_billions_component = ((graph_hundred) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_billions = billions @ graph_billions_component + + ten_billions = NEMO_DIGIT**12 + graph_ten_billions_component = ((graph_thousand) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**13 + graph_hundred_billions_component = (graph_digit + pynutil.insert('조')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + trillion = NEMO_DIGIT**14 + graph_trillion_component = ( + (NEMO_DIGIT**2 @ graph_1_to_99) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_trillions = trillion @ graph_trillion_component + + ten_trillions = NEMO_DIGIT**15 + graph_ten_trillions_component = ( + (graph_hundred) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_ten_trillions = ten_trillions @ graph_ten_trillions_component + + hundred_trillions = NEMO_DIGIT**16 + graph_hundred_trillions_component = ( + (graph_thousand) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_hundred_trillions = hundred_trillions @ graph_hundred_trillions_component + + thousand_trillions = NEMO_DIGIT**17 + graph_thousand_trillions_component = ( + graph_digit + + pynutil.insert('경') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_trillions_component, + pynutil.delete('0') + graph_ten_trillions_component, + pynutil.delete('00') + graph_trillion_component, + pynutil.delete('000') + graph_hundred_billions_component, + pynutil.delete('0000') + graph_ten_billions_component, + pynutil.delete('00000') + graph_billions_component, + pynutil.delete('000000') + graph_thousand_million_component, + pynutil.delete('0000000') + graph_hundred_million_component, + pynutil.delete('00000000') + graph_ten_million_component, + pynutil.delete('000000000') + graph_million_component, + pynutil.delete('0000000000') + graph_hundred_thousand_component, + pynutil.delete('00000000000') + graph_ten_thousand_component, + pynutil.delete('000000000000') + graph_thousand_component, + pynutil.delete('0000000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_thousand_trillions = thousand_trillions @ graph_thousand_trillions_component + + # FST + graph_num = pynini.union( + graph_thousand_trillions, + graph_hundred_trillions, + graph_ten_trillions, + graph_trillions, + graph_hundred_billions, + graph_ten_billions, + graph_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_1_to_99, + graph_zero, + ).optimize() + + # ---------------------------- + # Native counting + counters + # e.g., 3개, 2명, 10살 + # + # In Korean, counters require native numeral forms + # for small numbers (한/두/세…, 열/스무/서른…). + counter_suffix = pynini.string_file(get_abs_path("data/number/counter_suffix.tsv")) + counter_suffix_accep = pynini.project(counter_suffix, "input").optimize() + + native_ones = pynini.string_file(get_abs_path("data/number/native_ones.tsv")) # 1~9: 한/두/세/... + ordinal_tens = pynini.string_file(get_abs_path("data/ordinal/tens.tsv")) # 10=열, 20=스무, 30=서른 + ordinal_tens_prefix = pynini.string_file(get_abs_path("data/ordinal/tens_prefix.tsv")) # 열/스물/서른 + + native_11_to_39 = (ordinal_tens_prefix + native_ones).optimize() + native_1_to_39 = pynini.union(native_ones, ordinal_tens, native_11_to_39).optimize() + + # Compose number + counter as one cardinal token + counter_case = ( + pynutil.insert('integer: "') + + native_1_to_39 + + pynutil.insert('" ') + + pynutil.insert('counter: "') + + counter_suffix_accep + + pynutil.insert('"') + ).optimize() + + # Sign and final formatting + # Build the integer token (integer: "...") + integer_token = pynutil.insert('integer: "') + graph_num + pynutil.insert('"') + + # Sign handling: + # - minus sets negative flag + # - plus is ignored (positive number) + minus_prefix = pynutil.insert('negative: "true" ') + pynutil.delete("-") + plus_prefix = pynutil.delete("+") + + # Accounting negative: "( 1,234 )" -> negative + integer:"1234" + paren_negative = ( + pynutil.insert('negative: "true" ') + pynutil.delete("(") + ws + integer_token + ws + pynutil.delete(")") + ) + + # Signed number: optional (+|-) + integer + signed_integer = (minus_prefix | plus_prefix).ques + integer_token + + # Prefer accounting-form first, then signed form + final_graph = paren_negative | signed_integer | counter_case + + # Wrap with class tokens and finalize + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + self.graph = graph_num diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe9df4ee6841dc6bb9e7c20351557c13f4f9173 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -0,0 +1,310 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying dates in Korean, e.g. + 2024/01/30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024/1/30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024-01-30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024.01.30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + + 기원전233년 -> date { era: "기원전" year: "이백삼십삼년" } + 기원후2024년 -> date { era: "기원후" year: "이천이십사년" } + + 21일월요일 -> tokens { date { day: "이십일일" weekday: "월요일" } } + 1970년대 -> date { year: "천구백칠십년대" } + + 1월1일(월)~3일(수) + -> tokens { date { month: "일월" day: "일일" weekday: "월요일" } } + tokens { name: "부터" } + tokens { date { day: "삼일" weekday: "수요일" } } + + 1970~1980년대 + -> tokens { cardinal { integer: "천구백칠십" } } + tokens { name: "부터" } + tokens { date { year: "천구백팔십년대" } } + + 7월5~9일(월~금) + -> tokens { date { month: "칠월" } } + tokens { cardinal { integer: "오" } } + tokens { name: "부터" } + tokens { date { day: "구일" weekday: "월요일" } } + tokens { name: "부터" } + tokens { date { weekday: "금요일" } } + + 2023년3월1일(수)~6월12일(화) + -> tokens { date { year: "이천이십삼년" month: "삼월" day: "일일" weekday: "수요일" } } + tokens { name: "부터" } + tokens { date { month: "유월" day: "십이일" weekday: "화요일" } } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + strip0 = pynini.closure(pynutil.delete("0"), 0, 1) + graph_cardinal = cardinal.graph + cardinal_lz = (strip0 + graph_cardinal).optimize() + + # Load base .tsv files + week = pynini.string_file(get_abs_path("data/date/week.tsv")) + month_exceptions = pynini.string_file(get_abs_path("data/date/exceptions.tsv")) + month_exceptions_inputs = pynini.project(month_exceptions, "input").optimize() + + # Non-exception inputs go through the generic cardinal path + graph_cardinal_non_exceptions = pynini.compose( + pynini.difference(pynini.project(graph_cardinal, "input"), month_exceptions_inputs).optimize(), + graph_cardinal, + ).optimize() + + # Month cardinal: prefer exceptions; + month_cardinal = strip0 + (month_exceptions | graph_cardinal_non_exceptions).optimize() + + era = pynini.union("기원전", "기원후").optimize() + signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + + # Strict digit ranges for M/D/Y and Y/M/D + _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)]) + _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)]) + + # For standalone years: + # - No era: 1–4 digits with NO leading zeros + YEAR_NO_ERA_1TO4 = pynini.closure(pynutil.delete("0"), 0, 3) + _1to9 + pynini.closure(_d, 0, 3) + # - With era (기원전/기원후): allow leading zeros but strip them + YEAR_ERA_1TO4 = pynini.closure(pynutil.delete("0"), 0, 3) + _1to9 + pynini.closure(_d, 0, 3) + + # MM: 01-09 | 10-12 + MM = (pynini.accep("0") + _1to9) | (pynini.accep("1") + pynini.union("0", "1", "2")) + + # DD: 01-09 | 10-19 | 20-29 | 30-31 + DD = ( + (pynini.accep("0") + _1to9) + | (pynini.accep("1") + _d) + | (pynini.accep("2") + _d) + | (pynini.accep("3") + pynini.union("0", "1")) + ) + + # YYYY: exactly 4 digits and two-digit year for M/D/YY and D/M/YY + YYYY = pynini.union("1", "2") + _d + _d + _d + YY = _d + _d + + # Map digits -> cardinal words using existing graphs (strip leading zero via month_cardinal/cardinal_lz) + mm_to_text = pynini.compose(MM, month_cardinal).optimize() + dd_to_text = pynini.compose(DD, cardinal_lz).optimize() + yy_to_text = pynini.compose(YY, graph_cardinal).optimize() + + # Components with tags/suffixes (strict) + month_component_md = ( + pynutil.insert("month: \"") + mm_to_text + pynutil.insert("월") + pynutil.insert("\"") + ).optimize() + day_component_md = ( + pynutil.insert("day: \"") + dd_to_text + pynutil.insert("일") + pynutil.insert("\"") + ).optimize() + year_component_y2 = ( + pynutil.insert("year: \"") + yy_to_text + pynutil.insert("년") + pynutil.insert("\"") + ).optimize() + + # Generic components + era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") + + # Brackets for weekday + front_bracket = ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_space)) + ) | ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_space)) + ) + preceding_bracket = ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_space)) + ) | ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_space)) + ) + + week_component_bracketed = ( + (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynini.cross("〜", "부터") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynutil.delete("・") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + ) + + week_component_plain = pynutil.insert("weekday: \"") + week + pynutil.insert("\"") + week_component = week_component_bracketed | week_component_plain + + # Strict 4-digit year component (1000–2999) + year_component_y4_strict = ( + pynutil.insert("year: \"") + (YYYY @ graph_cardinal) + pynutil.insert("년") + pynutil.insert("\"") + ).optimize() + + # Prefer strict 4-digit; still allow 2-digit with worse weight (for MM/DD/YY etc.) + year_component_md_strict = (year_component_y4_strict | pynutil.add_weight(year_component_y2, 1.0)).optimize() + + # Format: YYYY/MM/DD(weekday) + graph_basic_date = ( + pynini.closure(era_component + insert_space, 0, 1) + + year_component_y4_strict + + signs + + insert_space + + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) + + signs + + insert_space + + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) + + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + ) + + # American: MM/DD/YYYY + graph_american_date = ( + month_component_md + + signs + + insert_space + + day_component_md + + signs + + insert_space + + year_component_md_strict + + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + ).optimize() + + # European: DD/MM/YYYY + graph_european_date = ( + day_component_md + + signs + + insert_space + + month_component_md + + signs + + insert_space + + year_component_md_strict + + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + ).optimize() + + # Single elements (year/month/day) + individual_year_component = ( + # with era: (기원전|기원후) + 1~4 digits (leading zeros allowed → stripped) + ( + era_component + + insert_space + + pynutil.insert("year: \"") + + (YEAR_ERA_1TO4 @ graph_cardinal) + + pynini.accep("년") + + pynutil.insert("\"") + ) + | + # no era: 1~4 digits, no leading zero + ( + pynutil.insert("year: \"") + + (YEAR_NO_ERA_1TO4 @ graph_cardinal) + + pynini.accep("년") + + pynutil.insert("\"") + ) + ).optimize() + + individual_month_component = ( + pynutil.insert("month: \"") + month_cardinal + pynini.accep("월") + pynutil.insert("\"") + ) + + month_josa = pynini.union("에", "은", "는", "에는") + + individual_month_component_with_josa = ( + pynutil.insert('month: "') + + month_cardinal + + pynini.accep("월") + + pynini.closure(month_josa, 0, 1) + + pynutil.insert('"') + ).optimize() + + individual_day_component = pynutil.insert("day: \"") + cardinal_lz + pynini.accep("일") + pynutil.insert("\"") + + week_full_word_acceptor = pynini.project(week, "output") + week_component_full_word = pynutil.insert("weekday: \"") + week_full_word_acceptor + pynutil.insert("\"") + + day_and_weekday_component = ( + individual_day_component + pynini.closure(insert_space, 0, 1) + week_component_full_word + ) + + month_and_weekday_component = ( + individual_month_component + pynini.closure(insert_space, 0, 1) + week_component_full_word + ) + + graph_individual_component = ( + day_and_weekday_component + | month_and_weekday_component + | individual_year_component + | individual_month_component_with_josa + | individual_month_component + | individual_day_component + | week_component + ) + pynini.closure(insert_space + week_component, 0, 1) + + graph_individual_component_combined = ( + (individual_year_component + insert_space + individual_month_component) + | (individual_month_component + insert_space + individual_day_component) + | ( + individual_year_component + + insert_space + + individual_month_component + + insert_space + + individual_day_component + ) + ) + pynini.closure(insert_space + week_component, 0, 1) + + nendai = pynini.accep("년대") + era_nendai = ( + pynini.closure(era_component + insert_space, 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + nendai + + pynutil.insert("\"") + ).optimize() + + graph_all_date = ( + graph_basic_date + | graph_american_date + | graph_european_date + | graph_individual_component + | graph_individual_component_combined + | era_nendai + ).optimize() + + final_graph = self.add_tokens(graph_all_date) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..bda1174aa08c28fc1406fdfd6f81f9a12fd6b5ca --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/decimal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal numbers in Korean, e.g. + 1.23 -> decimal { integer_part: "일" fractional_part: "이삼" } + -0.5 -> decimal { negative: "마이너스" integer_part: "영" fractional_part: "오" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + # Use the base cardinal graph for the integer part + base_integer_graph = cardinal.graph + # Only special-case 10000 -> 만 for decimal integer part (if needed) + specials_input = pynini.cross("10000", "만") + + # Try the special mapping first, then fall back to normal cardinal + cardinal_before_decimal = (specials_input | base_integer_graph).optimize() + + cardinal_after_decimal = pynini.string_file(get_abs_path("data/number/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + DOUBLE_QUOTE = '"' + + graph_integer = ( + pynutil.insert(f'integer_part: {DOUBLE_QUOTE}') + cardinal_before_decimal + pynutil.insert(DOUBLE_QUOTE) + ) + graph_fractional = ( + pynutil.insert(f'fractional_part: {DOUBLE_QUOTE}') + + pynini.closure(cardinal_after_decimal | zero, 1) + + pynutil.insert(DOUBLE_QUOTE) + ) + + # Decimal without a sign (e.g., 2.5) + graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(NEMO_SPACE) + graph_fractional + + # Negative sign handling (e.g., -2.5 or 마이너스2.5) + graph_with_negative = ( + pynutil.insert(f'negative: {DOUBLE_QUOTE}') + + (pynini.cross("-", "마이너스") | pynini.accep("마이너스")) + + pynutil.insert(DOUBLE_QUOTE) + ) + + graph_decimal = graph_decimal_no_sign | ( + graph_with_negative + pynutil.insert(NEMO_SPACE) + graph_decimal_no_sign + ) + + # For internal use without tokens + self.just_decimal = graph_decimal_no_sign.optimize() + + # Final graph with tokens + final_graph = self.add_tokens(graph_decimal) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..3ef82e1ece7e805579603423764e064b142e30c2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/electronic.py @@ -0,0 +1,168 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer (FST) for classifying **electronic expressions** such as + email addresses, URLs, and domain names in Korean. + + Example conversions: + - abc@nvidia.co.kr → electronic { username: "abc" domain: "nvidia.co.kr" } + - www.nvidia.com → electronic { domain: "www.nvidia.com" } + - https://nvidia.com → electronic { protocol: "HTTPS colon slash slash" domain: "nvidia.com" } + - 1234-5678-9012-3456 → electronic { protocol: "credit card" domain: "1234567890123456" } + + Args: + cardinal: FST for digit/number verbalization (used for numeric parts if non-deterministic). + deterministic: If True, provides a single transduction path; otherwise allows multiple. + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + # ---------- Basic character ranges and symbols ---------- + LOWER = pynini.union(*[pynini.accep(c) for c in "abcdefghijklmnopqrstuvwxyz"]) + UPPER = pynini.union(*[pynini.accep(c) for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"]) + ASCII_LETTER = (LOWER | UPPER).optimize() + ASCII_ALNUM = (ASCII_LETTER | NEMO_DIGIT).optimize() + + HYPHEN = pynini.accep("-") + DOT = pynini.accep(".") + SLASH = pynini.accep("/") + AT = pynini.accep("@") + + # Handle numeric reading mode (only for non-deterministic mode) + numbers = ( + NEMO_DIGIT + if deterministic + else (pynutil.insert(NEMO_SPACE) + cardinal.long_numbers + pynutil.insert(NEMO_SPACE)) + ) + + # ---------- Load resources ---------- + cc_cues = pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")) + accepted_symbols = pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "input") + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + # ---------- Username ---------- + # Exclude '@' from username + username_symbols = pynini.difference(accepted_symbols, AT) + # Start with alphanumeric and allow symbols/numbers repeatedly + username_core = ASCII_ALNUM + pynini.closure(ASCII_ALNUM | numbers | username_symbols) + username = pynutil.insert('username: "') + username_core + pynutil.insert('"') + pynini.cross("@", NEMO_SPACE) + + # ---------- Domain ---------- + # Simplified RFC: label = [A-Za-z0-9-]+ , TLD = '.' [A-Za-z0-9]{2,} + label = pynini.closure(ASCII_ALNUM | HYPHEN, 1) + tld = DOT + pynini.closure(ASCII_ALNUM, 2) + # Domain can be (label + TLD) or TLD only (e.g., ".com") + domain_core = (label + pynini.closure(tld, 1)) | tld + + # Optional path after domain (e.g., /path) + path_segment = pynini.closure(NEMO_NOT_SPACE, 1) # at least one non-space character + path = SLASH + path_segment # / + optional_path = pynini.closure(path, 0, 1) # optional path + + domain_with_opt_path = domain_core + optional_path + + domain_graph_with_class_tags = ( + pynutil.insert('domain: "') + domain_with_opt_path.optimize() + pynutil.insert('"') + ) + + # ---------- protocol ---------- + protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "colon")) + pynutil.insert(NEMO_SPACE)) + protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( + pynini.accep("://") @ protocol_symbols + ) + protocol_file_start = pynini.accep("file") + insert_space + (pynini.accep(":///") @ protocol_symbols) + protocol_end = pynutil.add_weight(pynini.cross("www", "WWW ") + pynini.accep(".") @ protocol_symbols, -1000) + protocol = protocol_file_start | protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') + + # ---------- Combine all graphs ---------- + graph = pynini.Fst() # empty + + # (1) Email pattern + email_guard = NEMO_SIGMA + AT + NEMO_SIGMA + DOT + NEMO_SIGMA + graph |= pynini.compose(email_guard, username + domain_graph_with_class_tags) + + # (2) Domain only (without protocol)) + # Domain core graph + graph_domain = (pynutil.insert('domain: "') + domain_core + pynutil.insert('"')).optimize() + graph |= graph_domain + + known_extensions = pynini.project( + pynini.string_file(get_abs_path("data/electronic/extensions.tsv")), + "input", + ) + + filename_stem = pynini.closure( + pynini.difference(NEMO_NOT_SPACE, pynini.union(SLASH, DOT)), + 1, + ) + + file_with_extension = filename_stem + known_extensions + + graph |= (pynutil.insert('domain: "') + file_with_extension + pynutil.insert('"')).optimize() + + # (3) URL with protocol + graph |= protocol + insert_space + domain_graph_with_class_tags + + # (4) Credit card pattern: cue + 4–16 digits + if deterministic: + cc_digits = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + + cc_cues + + pynutil.insert('" domain: "') + + delete_space + + cc_digits + + pynutil.insert('"') + ) + graph |= cc_phrases + + four = pynini.closure(NEMO_DIGIT, 4, 4) + sep_token = pynini.union(HYPHEN, NEMO_SPACE) + cc16_grouped = (four + pynini.cross(sep_token, " ")) ** 3 + four + cc16_grouped = cc16_grouped + delete_space + + cc16_no_cue = ( + pynutil.insert('protocol: "신용카드 " ') + + pynutil.insert('domain: "') + + cc16_grouped + + pynutil.insert('"') + ) + + # Give it higher priority over Date FST + cc16_no_cue = pynutil.add_weight(cc16_no_cue.optimize(), -1.0) + + graph |= cc16_no_cue + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..01159141c3b52f50e015d947cbc86be26bd98c62 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/fraction.py @@ -0,0 +1,113 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying Korean fractions, e.g. + 3/5 → tokens { fraction { numerator: "삼" denominator: "오" } } + 2과7/9 → tokens { fraction { integer_part: "이" numerator: "칠" denominator: "구" } } + 마이너스3/5 → tokens { fraction { negative: "마이너스" numerator: "삼" denominator: "오" } } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal = cardinal.graph + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + DOUBLE_QUOTE = '"' + slash = pynutil.delete('/') + root = pynini.accep('√') + + # Decimal number (e.g., 1.23 → 일점이삼) + decimal_number = cardinal + pynini.cross(".", "점") + pynini.closure(graph_digit | graph_zero) + + # Accept cardinal / root + cardinal / decimal / root + decimal + numeral = cardinal | (root + cardinal) | decimal_number | (root + decimal_number) + + # Integer part (e.g., 2과, 1와) + integer_component = ( + pynutil.insert(f'integer_part: {DOUBLE_QUOTE}') + + numeral + + (pynini.accep("과") | pynini.accep("와")) + + pynutil.insert(DOUBLE_QUOTE) + ) + + integer_component_with_space = integer_component + delete_space + pynutil.insert(NEMO_SPACE) + + # Denominator and numerator + denominator_component = pynutil.insert(f'denominator: {DOUBLE_QUOTE}') + numeral + pynutil.insert(DOUBLE_QUOTE) + + numerator_component = pynutil.insert(f'numerator: {DOUBLE_QUOTE}') + numeral + pynutil.insert(DOUBLE_QUOTE) + + # Format 1: 3/4 style + graph_fraction_slash = ( + pynini.closure(integer_component_with_space, 0, 1) + + numerator_component + + slash + + pynutil.insert(NEMO_SPACE) + + denominator_component + + pynutil.insert(NEMO_SPACE) + + pynutil.insert('morphosyntactic_features: "분의"') + ) + + # Format 2: Korean native "4분의3" style + graph_fraction_word = ( + pynini.closure(integer_component_with_space, 0, 1) + + denominator_component + + pynutil.delete("분의") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert('morphosyntactic_features: "분의"') + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + # Optional particles following the fraction + particle_subject = pynutil.insert('morphosyntactic_features: "분의_subject"') + ( + pynutil.delete("이") | pynutil.delete("가") + ) + particle_topic = pynutil.insert('morphosyntactic_features: "분의_topic"') + ( + pynutil.delete("은") | pynutil.delete("는") + ) + particle_object = pynutil.insert('morphosyntactic_features: "분의_object"') + ( + pynutil.delete("을") | pynutil.delete("를") + ) + + optional_particle = pynini.closure( + pynutil.insert(NEMO_SPACE) + (particle_subject | particle_topic | particle_object), + 0, + 1, + ) + + # Optional minus sign + optional_sign = ( + pynutil.insert(f'negative: {DOUBLE_QUOTE}') + + (pynini.accep("마이너스") | pynini.cross("-", "마이너스")) + + pynutil.insert(DOUBLE_QUOTE) + + pynutil.insert(NEMO_SPACE) + ) + + # Combine full graph + graph = pynini.closure(optional_sign, 0, 1) + (graph_fraction_slash | graph_fraction_word) + optional_particle + self.graph = graph.optimize() + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..70a267e43300e2dc5c8a1366539623d3010fc35d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/measure.py @@ -0,0 +1,106 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying Korean measure expressions. + - 1kg → measure { cardinal { integer: "일" } units: "킬로그램" } + - 12.5km → measure { decimal { integer_part: "십이" fractional_part: "오" } units: "킬로미터" } + - 2/3m → measure { fraction { numerator: "이" denominator: "삼" } units: "미터" } + - 60km/h → measure { cardinal { integer: "육십" } units: "킬로미터 퍼 시간" } + + This FST attaches measurement units (e.g., "킬로미터", "그램") to numeric expressions + classified by the `cardinal`, `decimal`, or `fraction` subgraphs. + + Args: + cardinal: FST handling integer (cardinal) numbers. + decimal: FST handling decimal numbers (optional). + fraction: FST handling fractional numbers (optional). + deterministic: If True, provides a single transduction path; otherwise allows multiple. + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst = None, + fraction: GraphFst = None, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + # Numeric subgraphs + graph_cardinal = cardinal.graph + + # Unit lexicon + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) + + # Per-expression handling (e.g., km/h, m/s) + opt_space = pynini.closure(delete_space, 0, 1) + per = pynini.cross("/", "퍼") + opt_space + insert_space + graph_unit + optional_per = pynini.closure(opt_space + insert_space + per, 0, 1) + + # Final unit FST produces either "" or " 퍼 " + unit = pynutil.insert('units: "') + (graph_unit + optional_per | per) + pynutil.insert('"') + + minus_as_field = pynutil.insert('negative: "마이너스" ') + consume_minus = pynini.cross("-", "") | pynini.cross("마이너스", "") + + # Optional minus field + removal of actual sign symbol or word + optional_minus = pynini.closure(minus_as_field + consume_minus + opt_space, 0, 1) + + # Combine numeric and unit components + pieces = [] + + # 1) Cardinal form: e.g., "12kg" + sub_cardinal = ( + pynutil.insert("cardinal { ") + + pynutil.insert('integer: "') + + graph_cardinal + + delete_space + + pynutil.insert('" } ') + + unit + ) + pieces.append(sub_cardinal) + + # 2) Decimal form: e.g., "12.5km" + if decimal is not None: + sub_decimal = ( + pynutil.insert("decimal { ") + + optional_minus + + decimal.just_decimal + + delete_space + + pynutil.insert(" } ") + + unit + ) + pieces.append(sub_decimal) + + # 3) Fraction form: e.g., "2/3m" or "삼분의 이 미터" + if fraction is not None: + sub_fraction = pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit + pieces.append(sub_fraction) + + # Union all supported numeric forms (cardinal | decimal | fraction) + graph = pieces[0] + for p in pieces[1:]: + graph |= p + + # Final wrapping into tokens { measure { ... } } + self.fst = self.add_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..90146d6b35e84da3b9fdb80b2f6b04903b952217 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/money.py @@ -0,0 +1,99 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying Korean money. + + Example inputs and outputs: + ₩350 -> money { currency_maj: "원" integer_part: "삼백오십" } + 350원 -> money { integer_part: "삼백오십" currency_maj: "원" } + KRW 12,050 -> money { currency_maj: "원" integer_part: "일만이천오십" } + 12만 500원 -> money { integer_part: "십이만오백" currency_maj: "원" } + ₩10.25 -> money { currency_maj: "원" integer_part: "십" minor_part: "이십오" } # optional 2-digit minor + 0원 -> money { integer_part: "영" currency_maj: "원" } + + Args: + cardinal: CardinalFst + deterministic: If True, provide a single transduction; + if False, allow multiple transductions. + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.graph + sp = pynini.closure(delete_space) # absorb any amount of spaces in input + + # --- Numbers (integer / optional minor) --- + # Integer part: "0" or a non-zero leading digit; allow commas (e.g., 18,925,000) + integer_part_fst = pynini.union("0", (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT | pynutil.delete(","))) + + # Plain integer → integer_part: "" + graph_integer_plain = ( + pynutil.insert('integer_part: "') + (integer_part_fst @ graph_cardinal) + pynutil.insert('" ') + ) + + # Optional 2-digit decimal (kept as minor_part if ever used downstream) + decimal_part_fst = NEMO_DIGIT**2 + graph_minor = pynutil.insert('minor_part: "') + (decimal_part_fst @ graph_cardinal) + pynutil.insert('" ') + + # Integer with scale suffix (만/억/조) → wrap the whole thing in one integer_part + scale_unit = pynini.union("만", "억", "조") + value_with_scale = (integer_part_fst @ graph_cardinal) + scale_unit + graph_integer_with_suffix = ( + pynutil.insert('integer_part: "') + value_with_scale + pynutil.insert('" ') + ).optimize() + + # Integer (+ optional ".<2-digit>" minor) + number_component_plain = graph_integer_plain + pynini.closure(pynutil.delete(".") + graph_minor, 0, 1) + number_component = (graph_integer_with_suffix | number_component_plain).optimize() + + # --- Currency (prefix or suffix) --- + # currency_major.tsv example: + # ₩ 원 + # KRW 원 + # 원 원 + maj_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + + # Prefix currency (e.g., ₩, KRW): emit currency_maj then number + currency_major_prepended = pynini.union( + *[pynutil.delete(surface) + pynutil.insert(f'currency_maj: "{unit}" ') for surface, unit in maj_labels] + ).optimize() + + # Suffix currency (e.g., ...원, ...달러): convert unit literal to currency_maj + currency_major_appended = pynini.union( + *[pynutil.delete(unit) + pynutil.insert(f'currency_maj: "{unit}" ') for _, unit in maj_labels] + ).optimize() + + # --- Compose (NO period handling) --- + # NOTE: We deliberately do NOT consume '/월', '/년', '/주', '/일', '/시간' here. + # If present in the raw text, they remain outside the money token and can be handled upstream/elsewhere. + + # [currency] [number] + graph_prepend = (currency_major_prepended + sp + number_component).optimize() + + # [number] [currency] + graph_append = (number_component + currency_major_appended).optimize() + + graph = (graph_prepend | graph_append).optimize() + + self.fst = self.add_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..31e582d378dd335d0c74ef5da4849f7e9ef11651 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/ordinal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Korean ordinal expressions, e.g. + 1번째 -> ordinal { integer: "첫번째" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + # Load base .tsv files + graph_digit = pynini.string_file(get_abs_path("data/ordinal/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_exceptions = pynini.string_file(get_abs_path("data/ordinal/exceptions.tsv")) + graph_tens = pynini.string_file(get_abs_path("data/ordinal/tens.tsv")) + graph_tens_prefix = pynini.string_file(get_abs_path("data/ordinal/tens_prefix.tsv")) + + graph_11_to_39 = (graph_tens_prefix + graph_digit).optimize() + + # Combine all ordinal forms from 1 to 39 + graph_ordinal_1to39 = ( + graph_exceptions | graph_digit | graph_zero | graph_tens | graph_11_to_39 + ).optimize() + pynini.accep("번째") + + # Accept tens digit 4–9 + tens_digit_4_to_9_accep = pynini.union(*[pynini.accep(str(i)) for i in range(4, 10)]) + # Accept any single digit + any_single_digit_accep = pynini.union(*[pynini.accep(str(i)) for i in range(0, 10)]) + # Combine two digits + from_40_to_99_inputs = tens_digit_4_to_9_accep + any_single_digit_accep + + # Match numbers with 3 or more digits + input_100_plus = pynini.closure(any_single_digit_accep, 3) + + # Combine both ranges (40–99 and 100+): total range = 40 and above + filter_inputs_from_40 = (from_40_to_99_inputs | input_100_plus).optimize() + + # Only allow cardinal numbers that are 40 or more + graph_cardinal_from40_filtered = pynini.compose(filter_inputs_from_40, cardinal.graph) + + # Add "번째" to the filtered cardinal graph. + graph_ordinal_from40 = graph_cardinal_from40_filtered + pynini.accep("번째") + + graph_ordinal = (graph_ordinal_1to39 | graph_ordinal_from40).optimize() # Handles 1-39 # Handles 40+ + + # Single-character particles (가, 이, 은, 는, 로, 도 ...) + josa_single = pynini.union("가", "이", "은", "는", "를", "을", "로", "도", "다") + + # Multi-character particles (부터, 까지) + josa_multi = pynini.union("부터", "까지") + + # Allow patterns like: + # 번째 + (optional single-josa) + (optional multi-josa) + josa = (josa_single.ques + josa_multi.ques).optimize() + + # Final ordinal graph with optional particles + graph_ordinal_with_josa = (graph_ordinal + josa).optimize() + + # Build the “integer: …” token structure + final_graph = pynutil.insert('integer: "') + graph_ordinal_with_josa + pynutil.insert('"') + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/punctuation.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..3ade742d85a0cab0be07f0249a7fe617a97beea3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/punctuation.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + range_component = pynini.cross("〜", "부터") | pynini.accep("부터") + + graph = pynutil.insert('name: "') + range_component + pynutil.insert('"') + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..48ce849adfe8d1e9191e5c1432acb4fc41d895bd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -0,0 +1,89 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying Korean telephone numbers. + + Example inputs → tokens: + +82 010-3713-7050 -> telephone { country_code: "국가번호 팔이," number_part: "영일영 삼칠일삼 칠영오영" } + +1 (415) 555-0123 -> telephone { country_code: "국가번호 일," number_part: "사일오 오오오 영일이삼" } + (031)371-3700 -> telephone { number_part: "영삼일 삼칠일 삼칠영영" } + 010-3713-7050 -> telephone { number_part: "영일영 삼칠일삼 칠영오영" } + 010.777.8888 -> telephone { number_part: "영일영 칠칠칠 팔팔팔팔" } + + Args: + deterministic (bool, optional): If True, provide a single transduction; + if False, allow multiple transductions. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + # Separator between digit blocks (e.g., "-" or ".") + delete_sep = pynutil.delete("-") | pynutil.delete(".") + # Optional space inserted between blocks + insert_block_space = insert_space + + # 1) safe digit mapping: force 0 -> "영" (do not rely on zero.tsv invert) + digit = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + zero_map = pynini.cross("0", "영") + digit_ko = (digit | zero_map).optimize() + + three_digits = digit_ko**3 + four_digits = digit_ko**4 + + # country code: "+1", "+82", "+1-" + cc_digits = pynini.closure(digit_ko, 1, 3) + + country_code = ( + pynutil.delete("+") + + pynutil.insert('country_code: "') + + cc_digits + + pynutil.insert('"') + + pynini.closure(pynutil.delete("-") | pynutil.delete(" "), 0, 1) + + delete_space + ) + + # area part: "123-" | "123." | "(123)" [space?] or "(123)-" + area_core = three_digits + area_part = ( + (area_core + delete_sep) + | ( + pynutil.delete("(") + + area_core + + pynutil.delete(")") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynini.closure(delete_sep, 0, 1) + ) + ) + insert_block_space + + # 2) allow 3 **or 4** digits in the middle block (to support 010-3713-7050) + mid = pynini.union(three_digits, four_digits) + last4 = four_digits + + # consume '-' or '.' between middle and last blocks + number_part_core = area_part + mid + delete_sep + insert_block_space + last4 + number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') + + # final graph: with or without country code + graph = pynini.union(country_code + insert_space + number_part, number_part).optimize() + + self.fst = self.add_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..d395cede9d3344db07d31865046d68bc57b3f342 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/time.py @@ -0,0 +1,192 @@ +# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 오전 10시 30분 -> time { suffix: "오전" hours: "열시" minutes: "삼십분" } + 오후 3시 반 -> time { suffix: "오후" hours: "세시" minutes: "삼십분" } + 자정 -> time { hours: "영시" } + 정오 -> time { hours: "열두시" } + + Args: + cardinal: CardinalFst (Korean cardinal graph) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + # Base number-to-words graph from the Cardinal Fst + graph_cardinal = cardinal.graph + strip0 = pynini.closure(pynutil.delete("0"), 0, 1) + + SP = pynini.closure(delete_space) + SEP = SP + insert_space + hour_clock = pynini.string_file(get_abs_path("data/time/hour.tsv")) + division = pynini.string_file(get_abs_path("data/time/division.tsv")) + + division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + + # --- Special single-word times --- + noon = pynini.accep("정오") + midnight = pynini.accep("자정") + noon_component = pynutil.insert("hours: \"") + pynini.cross(noon, "열두시") + pynutil.insert("\"") + midnight_component = pynutil.insert("hours: \"") + pynini.cross(midnight, "영시") + pynutil.insert("\"") + + # --- Korean Hangul components (H시 [M분|반] [S초]) --- + # "0" or "00" -> 0 + h_zero = strip0 + pynini.accep("0") + # "13".."24" + h_13_24 = pynini.union(*[str(i) for i in range(13, 25)]) + + # "0시" -> "영시" + hour_component_ko_zero = ( + pynutil.insert("hours: \"") + + pynini.cross(h_zero, "영") + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # "13시..24시" -> Sino-Korean words (십삼/…/이십사) + 시 + hour_component_ko_13_24 = ( + pynutil.insert("hours: \"") + + (h_13_24 @ graph_cardinal) + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # "1시..12시" -> Native Korean words (한/두/세/네/…/열두) + 시 + hour_component_ko_1_12 = ( + pynutil.insert("hours: \"") + + (strip0 + hour_clock) + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # Priority: 13-24 > 0 > 1-12 to prevent partial matching errors + hour_component_ko = (hour_component_ko_13_24 | hour_component_ko_zero | hour_component_ko_1_12).optimize() + + # Minutes: number+"분" or "반" (approx. 30분). Allows optional '쯤|경' after minutes/반. + about_word = pynini.union("쯤", "경") + minute_number = ( + pynutil.insert("minutes: \"") + + (strip0 + graph_cardinal) + + pynutil.delete("분") + + pynutil.insert("분") + + pynutil.insert("\"") + ) + minute_half = ( + pynutil.insert("minutes: \"") + + pynutil.delete("반") + + pynutil.insert("반") + + pynini.closure(about_word, 0, 1) + + pynutil.insert("\"") + ) + minute_component_ko = (minute_half | minute_number).optimize() + + second_component_ko = ( + pynutil.insert("seconds: \"") + + (strip0 + graph_cardinal) + + pynutil.delete("초") + + pynutil.insert("초") + + pynutil.insert("\"") + ) + + # Allow suffix before or after + suffix_prefix_opt = pynini.closure(division_component + SEP, 0, 1) + suffix_postfix_opt = pynini.closure(SEP + division_component, 0, 1) + + # Hangul patterns + graph_hangul = ( + suffix_prefix_opt + + ( + hour_component_ko + | (hour_component_ko + SEP + minute_component_ko) + | (hour_component_ko + SEP + minute_component_ko + SEP + second_component_ko) + | minute_component_ko + | (minute_component_ko + SEP + second_component_ko) + | second_component_ko + ) + + suffix_postfix_opt + ).optimize() + + # Special words with optional suffix + graph_special = (suffix_prefix_opt + (noon_component | midnight_component) + suffix_postfix_opt).optimize() + + # --- Clock patterns: HH:MM[:SS] --- + colon = pynutil.delete(":") + + # Map 1-12 hours using native-Korean words, allowing an optional leading zero. + graph_hour_1_12 = ( + pynutil.insert("hours: \"") + (strip0 + hour_clock) + pynutil.insert("시") + pynutil.insert("\"") + ).optimize() + + # 0, 00, and 13-24 -> Sino-Korean words + hour_sino_val = ( + pynini.cross("00", "0") + | pynini.cross("0", "0") + | pynini.union(*[pynini.cross(str(i), str(i)) for i in range(13, 25)]) + ) + hour_sino_read = hour_sino_val @ graph_cardinal + + graph_hour_others = pynutil.insert("hours: \"") + hour_sino_read + pynutil.insert("시") + pynutil.insert("\"") + + hour_clock_component = (graph_hour_1_12 | graph_hour_others).optimize() + + minute_clock_component = ( + pynutil.insert("minutes: \"") + strip0 + graph_cardinal + pynutil.insert("분") + pynutil.insert("\"") + ) + second_clock_component = ( + pynutil.insert("seconds: \"") + strip0 + graph_cardinal + pynutil.insert("초") + pynutil.insert("\"") + ) + + # HH:MM (drop minutes if "00") + graph_hm_clock = ( + suffix_prefix_opt + + hour_clock_component + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + minute_clock_component, 0, 1)) + + suffix_postfix_opt + ).optimize() + + # HH:MM:SS (drop minutes/seconds if "00") + graph_hms_clock = ( + suffix_prefix_opt + + hour_clock_component + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + minute_clock_component, 0, 1)) + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + second_clock_component, 0, 1)) + + suffix_postfix_opt + ).optimize() + + graph = (graph_special | graph_hangul | graph_hm_clock | graph_hms_clock).optimize() + graph_final = self.add_tokens(graph) + self.fst = graph_final.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..73a49ab1d78c6f847d3399f8f5df1a8bdd63adcd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -0,0 +1,114 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str = "cased", + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ko_tn_{deterministic}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(cardinal=cardinal, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + word = WordFst(deterministic=deterministic) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 1.05), + pynutil.add_weight(word.fst, 100), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(telephone.fst, 1.1), + pynutil.add_weight(electronic.fst, 1.11), + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + + graph = ( + delete_space + token + pynini.closure((delete_extra_space | pynini.accep("")) + token) + delete_space + ) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..34367cb5eec3338a7c064806a746aaef351338fb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/whitelist.py @@ -0,0 +1,29 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class WhiteListFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = pynutil.insert('name: "') + whitelist + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..70217671e3b5986157d5c9d8e6ae3ebe6784925d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/taggers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Korean word. + e.g. 이름 -> tokens { name: "이름" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + word_char = pynini.difference(NEMO_NOT_SPACE, NEMO_DIGIT) + + word = pynutil.insert('name: "') + word += pynini.closure(word_char, 1) + word += pynutil.insert('"') + + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..51aaea3e82f550a6535ebab312dc7c8f616c6f6b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..bfadc12392e9e443a278dc0e5f641e250b2086be --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> 마이너스 이십삼 + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "마이너스 ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "음수 ") + self.optional_sign |= pynini.cross("negative: \"true\"", "- ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + # quoted: parses a quoted string value like "십", "명" + quoted = delete_space + pynutil.delete('"') + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + + # integer: mandatory field + integer = pynutil.delete("integer:") + quoted + + # counter: optional field (e.g., 명/개/살). + counter = pynutil.delete("counter:") + quoted + counter = pynini.closure(delete_space + counter, 0, 1) + self.numbers = self.optional_sign + integer + counter + + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..b0be9ced3a7a16f325df6cd4c5051cb17299818b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/date.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class DateFst(GraphFst): + """ + Korean date verbalizer + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + era_component = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + year_component = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + month_component = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + day_component = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + week_component = ( + pynutil.delete("weekday: \"") + + pynini.closure(delete_space) + + pynini.closure(NEMO_NOT_QUOTE) + + pynini.closure(delete_space) + + pynutil.delete("\"") + ) + + SPACE = pynini.closure(delete_space, 0, 1) + insert_space + + # This graph now correctly uses the 'delete_space' variable defined above. + graph_basic_date = ( + pynini.closure(era_component + SPACE, 0, 1) + + pynini.closure(year_component + SPACE, 0, 1) + + pynini.closure(month_component + SPACE, 0, 1) + + pynini.closure(day_component, 0, 1) + + pynini.closure(SPACE + week_component, 0, 1) + ) | (month_component + SPACE + week_component) + + final_graph = graph_basic_date + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd2a184233e833849d4a0251036bb648481625d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class DecimalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + # Extract integer part + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + # Extract fractional part and prepend "점" + fractional_part = ( + pynutil.delete('fractional_part: "') + + pynutil.insert("점") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # Verbalize decimal number without sign + decimal_positive = integer_part + pynutil.delete(" ") + fractional_part + + # Handle negative sign + negative_sign = ( + pynutil.delete('negative: "') + pynini.accep("마이너스") + pynutil.delete('"') + pynutil.delete(" ") + ) + + # Combine positive and negative cases + decimal = decimal_positive | (negative_sign + pynutil.insert(" ") + decimal_positive) + + delete_tokens = self.delete_tokens(decimal) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..7f4153b4dddfda682b2fa1f1b9888345a8f2ccfc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py @@ -0,0 +1,141 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_ALPHA, + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer (FST) for verbalizing **electronic expressions** (email/URL/domain). + + Input tokens: + tokens { electronic { username: "abc" domain: "abc.com" } } + + Example output (policy-dependent): + abc 골뱅이 abc 닷컴 + + Args: + deterministic: If True, produce a single verbalization. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + # 1) Handle digits (0–9) + graph_digit_no_zero = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + + graph_zero = pynini.cross("0", "영") + if not deterministic: + graph_zero |= pynini.cross("0", "공") + graph_digit = (graph_digit_no_zero | graph_zero).optimize() + + digit_inline_rewrite = pynini.cdrewrite( + graph_digit, + "", + "", + NEMO_SIGMA, + ) + + # 3) username part (add spaces between characters) + raw_username = pynini.closure(NEMO_NOT_QUOTE, 1) + + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete('"') + + (raw_username @ digit_inline_rewrite) + + pynutil.delete('"') + ) + + # 4) domain part (handle common endings like .com → 닷컴) + domain_common_pairs = ( + pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + | pynini.string_file(get_abs_path("data/electronic/extensions.tsv")) + ).optimize() + + # Rewrite known domains (.com → 닷컴) + tld_rewrite = pynini.cdrewrite( + domain_common_pairs, + "", + "", + NEMO_SIGMA, + ) + # Add a space before “닷” if needed + add_space_before_dot = pynini.cdrewrite( + pynini.cross("닷", " 닷"), + (NEMO_ALPHA | NEMO_DIGIT | NEMO_CHAR), + "", + NEMO_SIGMA, + ) + + raw_domain = pynini.closure(NEMO_NOT_QUOTE, 1) + + four = pynini.closure(NEMO_DIGIT, 4, 4) + cc16_grouped = four + pynutil.insert(" ") + four + pynutil.insert(" ") + four + pynutil.insert(" ") + four + cc_domain = (cc16_grouped @ digit_inline_rewrite).optimize() + + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete('"') + + ((raw_domain @ digit_inline_rewrite) @ tld_rewrite @ add_space_before_dot) + + delete_space + + pynutil.delete('"') + ).optimize() + + # 6) protocol (like “https://” or “file:///”) + protocol = ( + pynutil.delete('protocol: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + insert_space + ) + + protocol_raw = pynutil.delete('protocol: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + cc_protocol_guard = pynini.accep("신용카드") + pynini.closure(NEMO_NOT_QUOTE, 0) + cc_protocol = (protocol_raw @ cc_protocol_guard) + insert_space + + # Credit card case: "신용카드 ..." protocol + 16-digit domain grouped as 4-4-4-4 + cc_graph = ( + cc_protocol + + delete_space + + pynutil.delete("domain:") + + delete_space + + pynutil.delete('"') + + cc_domain + + pynutil.delete('"') + + delete_space + ).optimize() + + # 7) Combine: optional protocol + optional username + domain + default_graph = ( + pynini.closure(protocol + delete_space, 0, 1) + + pynini.closure(user_name + delete_space + pynutil.insert(" 골뱅이 ") + delete_space, 0, 1) + + domain + + delete_space + ).optimize() + + graph = (cc_graph | default_graph) @ pynini.cdrewrite(delete_extra_space, "", "", NEMO_SIGMA) + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..64c268b60f216154f643a3f952e7afe547545d0f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -0,0 +1,182 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing Korean fractions, e.g. + tokens { fraction { numerator: "3" denominator: "5" } } → 5분의3 + tokens { fraction { integer_part: "2" numerator: "7" denominator: "9" } } → 2과 9분의7 + tokens { fraction { denominator: "√8" numerator: "4" } } → 루트8분의4 + tokens { fraction { denominator: "2.75" numerator: "125" } } → 2.75분의125 + tokens { fraction { negative: "마이너스" numerator: "10" denominator: "11" } } → 마이너스11분의10 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + # Handles square root symbols like "√3" → "루트3" + denominator_root = pynini.cross("√", "루트") + pynutil.insert(NEMO_SPACE) + pynini.closure(NEMO_NOT_QUOTE) + numerator_root = pynini.cross("√", "루트") + pynutil.insert(NEMO_SPACE) + pynini.closure(NEMO_NOT_QUOTE) + + # Matches non-root numeric content + denominator = pynini.closure(NEMO_NOT_QUOTE - "√") + numerator = pynini.closure(NEMO_NOT_QUOTE - "√") + + # Delete FST field: denominator and extract value + denominator_component = ( + pynutil.delete('denominator: "') + (denominator_root | denominator) + pynutil.delete('"') + ) + numerator_component = pynutil.delete('numerator: "') + (numerator_root | numerator) + pynutil.delete('"') + + # Match fraction form: "denominator + 분의 + numerator" + # Also deletes optional morphosyntactic_features: "분의" if present + graph_fraction = ( + denominator_component + + pynutil.delete(NEMO_SPACE) + + pynini.closure( + pynutil.delete('morphosyntactic_features:') + delete_space + pynutil.delete('"분의"') + delete_space, + 0, + 1, + ) + + pynutil.insert("분의") + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + # Handle subject particle feature (분의_subject) + # Insert default particle "이" (will be corrected later via rewrite rules) + subject_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_subject"') + + delete_space + + pynutil.insert("이") # 일단 기본값 + ) + + # Handle topic particle feature (분의_topic) + topic_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_topic"') + + delete_space + + pynutil.insert("은") + ) + + # Handle object particle feature (분의_object) + object_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_object"') + + delete_space + + pynutil.insert("을") + ) + + # Combine fraction + optional particle suffix + # Particle is always inserted first in default form and later corrected + graph_fraction_all = graph_fraction + pynini.closure(subject_suffix | topic_suffix | object_suffix, 0, 1) + + # Handle integer + fraction (e.g., "2과 3/4") + # integer_part is removed and replaced with proper spacing + graph_integer = ( + pynutil.delete('integer_part:') + + delete_space + + pynutil.delete('"') + + pynini.closure(pynini.union("√", ".", NEMO_NOT_QUOTE - '"')) + + pynutil.delete('"') + + pynutil.insert(NEMO_SPACE) + ) + # Combine integer part with fraction + graph_integer_fraction = graph_integer + delete_space + graph_fraction_all + + # Handle optional negative prefix (e.g., "마이너스") + optional_sign = ( + pynutil.delete('negative:') + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE - '"') + + pynutil.delete('"') + + delete_space + + pynutil.insert(NEMO_SPACE) + ) + + # Final structure: + # [optional negative] + (integer + fraction OR fraction only) + graph = pynini.closure(optional_sign, 0, 1) + (graph_integer_fraction | graph_fraction_all) + + # Remove token wrappers + final_graph = self.delete_tokens(graph) + + # Sigma for rewrite context (entire string) + sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) + + # Fix subject particle agreement (이 → 가 for vowel-ending numerals) + # e.g., 사이 → 사가, 구이 → 구가 + subject_rewrite = pynini.cdrewrite( + pynini.string_map( + [ + ("이이", "이가"), + ("사이", "사가"), + ("오이", "오가"), + ("구이", "구가"), + ] + ), + "", + "", + sigma, + ) + + # Fix topic particle agreement (은 → 는) + # e.g., 이은 → 이는, 사은 → 사는 + topic_rewrite = pynini.cdrewrite( + pynini.string_map( + [ + ("이은", "이는"), + ("사은", "사는"), + ("오은", "오는"), + ("구은", "구는"), + ] + ), + "", + "", + sigma, + ) + + # Fix object particle agreement (을 → 를) + # e.g., 오을 → 오를, 이을 → 이를 + object_rewrite = pynini.cdrewrite( + pynini.string_map( + [ + ("이을", "이를"), + ("사을", "사를"), + ("오을", "오를"), + ("구을", "구를"), + ] + ), + "", + "", + sigma, + ) + + # Apply all rewrite rules sequentially and final optimized FST + final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..68384861c04ea310bf2590cb65d127d458d1f9e6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/measure.py @@ -0,0 +1,89 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing Korean measure tokens into surface text. + measure { cardinal { integer: "<...>" } units: "<...>" } + + Converts tokens like: + measure { cardinal { integer: "이" } units: "킬로그램" } + measure { fraction { numerator: "이" denominator: "삼" } units: "킬로미터" } + + into surface text: + "이 킬로그램", "삼분의 이 킬로미터" + + Args: + decimal, cardinal, fraction: Sub-verbalizers handling number types. + deterministic: If True, outputs a single normalized form. + """ + + def __init__( + self, + decimal: GraphFst = None, + cardinal: GraphFst = None, + fraction: GraphFst = None, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + # Combine all numeric verbalizers + graph_cardinal = cardinal.fst + graph_decimal = decimal.fst + graph_fraction = fraction.fst + + # Add a space after "마이너스" if it appears within numeric blocks + minus_space_rewrite = pynini.cdrewrite(pynini.cross("마이너스", "마이너스 "), "", "", NEMO_SIGMA).optimize() + + # Apply rewrite to each numeric subgraph to ensure spacing after "마이너스" + cardinal_spaced = graph_cardinal @ minus_space_rewrite + fraction_spaced = graph_fraction @ minus_space_rewrite + decimal_spaced = graph_decimal @ minus_space_rewrite + + # Combine all supported numeric types (cardinal | decimal | fraction) + number_block = decimal_spaced | cardinal_spaced | fraction_spaced + + # Extract and output unit string + units = ( + delete_space + + pynutil.delete("units:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # Normal form: + main = number_block + insert_space + units + + # preserve_order form: + preserve_order = delete_space + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + alt = units + insert_space + number_block + pynini.closure(preserve_order) + + graph = main | alt + + # Merge and clean tokens + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..f2c1112d2f83fcff86359c15f7761f40d347058e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/money.py @@ -0,0 +1,110 @@ +# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + +# ===== whitespace & token helpers ===== +sp = pynini.closure(delete_space) # absorb 0+ spaces +FIELD_VAL = pynini.closure(NEMO_NOT_QUOTE, 1) + + +def del_key_val(key: str): + """ + Delete the token field prefix and quotes, keep only the value. + + Input format: [sp] key: "" + Output: + + Example: + input 'integer_part: "삼백오십"' + output '삼백오십' + """ + return (sp + pynutil.delete(f'{key}: "') + FIELD_VAL + pynutil.delete('"')).optimize() + + +def drop_key_val(key: str): + """ + Delete the entire key-value pair (key and its quoted value). + + Input format: [sp] key: "" + Output: (nothing) + + Example: + input 'minor_part: "십"' + output '' + """ + return (sp + pynutil.delete(f'{key}: "') + FIELD_VAL + pynutil.delete('"')).optimize() + + +def drop_key_exact(key: str, val: str): + """ + Delete the exact key-value pair if it matches the given value. + + Input format: [sp] key: "val" + Output: (nothing) + + Example: + input 'currency_maj: "원"' + output '' + """ + return (sp + pynutil.delete(f'{key}: "{val}"')).optimize() + + +class MoneyFst(GraphFst): + """ + Verbalize Korean money. + + Input tokens: + tokens { money { integer_part: "..." currency_maj: "..." [minor_part: "..."] } } + + Period (e.g., /월, /년, …) is intentionally NOT handled here. + Output examples: + integer_part: "십" currency_maj: "원" -> "십원" + integer_part: "삼십억" currency_maj: "원" -> "삼십억원" + integer_part: "이백" currency_maj: "달러" -> "이백 달러" + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + # --- fields --- + integer_part = del_key_val("integer_part") + minor_part_drop = drop_key_val("minor_part") # ignore minor for KRW + currency_val_any = del_key_val("currency_maj") # ex) "원", "달러", "유로" + won_key_drop = drop_key_exact("currency_maj", "원") # don't print the key for KRW + + # ===== KRW (원) ===== + # (A) [integer] [원] -> "{integer}원" + won_a = integer_part + sp + won_key_drop + pynutil.insert("원") + # (B) [원] [integer] -> "{integer}원" + won_b = won_key_drop + sp + integer_part + pynutil.insert("원") + won_core = won_a | won_b + won_core = (won_core + pynini.closure(minor_part_drop, 0, 1)).optimize() + + # ===== Other currencies ===== + # "{integer} {currency}" (KRW sticks; others are spaced) + other_core = (integer_part + insert_space + currency_val_any).optimize() + other_core = (other_core + pynini.closure(minor_part_drop, 0, 1)).optimize() + + # ===== combine (no period) ===== + graph_core = (pynutil.add_weight(won_core, 0.0) | pynutil.add_weight(other_core, 0.5)).optimize() + + # no trailing period mapping + graph = graph_core + + # strip tokens wrapper + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..e801e57c9be53438c5a20958b61a19c8ce15b5ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing korean ordinal e.g. + tokens { ordinal { integer: "1번째" } } -> 첫번째 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_integer = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = graph_integer + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..fdf54528a74973dc8044aeaa7e09b53447e2c6ff --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py @@ -0,0 +1,83 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing Korean telephone numbers. + + Input: + telephone { [country_code: "...",] number_part: "..." [extension: "..."] } + Output: + [country_code + " "] + number_part [+ ", 내선 " + extension] + + Examples: + telephone { country_code: "국가번호 팔이," number_part: "영일영 삼칠일삼 칠영오영" } + -> 플러스 팔 이, 영일영, 삼칠일삼, 칠영오영 + telephone { number_part: "팔영영 오오오 영영영영" extension: "이삼사" } + -> 팔영영, 오오오, 영영영영, 내선 이삼사 + + Args: + deterministic: if True provides a single transduction; if False allows multiple. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + # country_code (optional, add trailing space if present) + country = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('country_code: "') + + pynutil.insert("국가번호 ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + insert_space + ) + + # number_part (mandatory) + number = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('number_part: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # extension (optional, prepend with ", 내선 ") + ext_field = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('extension: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + extension_opt = pynini.closure(pynutil.insert(", 내선 ") + ext_field, 0, 1) + + # remove wrapper "telephone { ... }" + graph = ( + pynutil.delete("telephone") + + pynini.closure(delete_space, 0, 1) + + pynutil.delete("{") + + pynini.closure(country, 0, 1) + + number + + extension_opt + + pynini.closure(delete_space, 0, 1) + + pynutil.delete("}") + ) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5e54e7f5fdc626e092865fcb0e5993691aae56 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/time.py @@ -0,0 +1,63 @@ +# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time e.g. + + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_content = pynini.closure(NEMO_NOT_QUOTE) + minute_component = pynutil.delete("minutes: \"") + pynini.cross("영분", "") + pynutil.delete( + "\"" + ) | pynutil.delete("minutes: \"") + (minute_content - "영분") + pynutil.delete("\"") + + second_content = pynini.closure(NEMO_NOT_QUOTE) + second_component = pynutil.delete("seconds: \"") + pynini.cross("영초", "") + pynutil.delete( + "\"" + ) | pynutil.delete("seconds: \"") + (second_content - "영초") + pynutil.delete("\"") + + division_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_basic_time = pynini.closure(division_component + delete_space + insert_space, 0, 1) + ( + ( + hour_component + + delete_space + + insert_space + + minute_component + + delete_space + + insert_space + + second_component + ) + | (hour_component + delete_space + insert_space + minute_component) + | hour_component + | minute_component + | second_component + ) + + final_graph = graph_basic_time + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c6cb0109899a8207652bc766e39a94a77bc001 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py @@ -0,0 +1,74 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + money = MoneyFst(deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + + graph = pynini.union( + cardinal.fst, + ordinal.fst, + word.fst, + decimal.fst, + fraction.fst, + date.fst, + whitelist.fst, + time.fst, + money.fst, + telephone.fst, + measure.fst, + electronic.fst, + ) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..64eb6ff28ccf1240c6f45fc11967e0863f6c902d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ko_tn_{deterministic}_verbalizer.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + + space_between_tokens = pynini.closure(NEMO_WHITE_SPACE, 1) + + verbalizer = ( + delete_space + + token_verbalizer + + pynini.closure(space_between_tokens + token_verbalizer) + + delete_space + ) + + self.fst = verbalizer.optimize() + + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..62c4d242e14d7489f661995d8c56124274d54e27 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ko.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + tokens { name: "부터" } -> 부터 + tokens { name: "~" } -> ~ + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..ba2feb0f16c5d4f40d237e803304e458f1fbc383 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/ko/verbalizers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + """ + Korean verbalizer for word. + tokens { name: "이름" } -> 이름 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/normalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/normalize.py new file mode 100644 index 0000000000000000000000000000000000000000..c73988c4968b762e3ae1339fab09747fd8e51fc1 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/normalize.py @@ -0,0 +1,851 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import json +import os +import re +import shutil +import sys +from argparse import ArgumentParser +from collections import OrderedDict +from glob import glob +from math import factorial +from time import perf_counter +from typing import Dict, List, Optional, Union + +import pynini +import regex +import tqdm +from joblib import Parallel, delayed +from pynini.lib.rewrite import top_rewrite +from sacremoses import MosesDetokenizer +from tqdm import tqdm + +from zonos2.vendor.nemo_text_processing.text_normalization.data_loader_utils import ( + load_file, + post_process_punct, + pre_process, + write_file, +) +from zonos2.vendor.nemo_text_processing.text_normalization.preprocessing_utils import additional_split +from zonos2.vendor.nemo_text_processing.text_normalization.token_parser import PRESERVE_ORDER_KEY, TokenParser +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +# this is to handle long input +sys.setrecursionlimit(3000) + +SPACE_DUP = re.compile(' {2,}') + + +""" +To normalize a single entry: + python normalize.py --text= + +To normalize text in .json manifest: + + python normalize.py \ + --input_file= \ + --output_file= \ + --n_jobs=-1 \ + --batch_size=300 \ + --manifest_text_field="text" \ + --whitelist= + + For a complete list of optional arguments, run: + >>> python normalize.py --help + + +To integrate Normalizer in your script: + >>> from zonos2.vendor.nemo_text_processing.text_normalization.normalize import Normalizer + # see the script for args details + >>> normalizer_en = Normalizer(input_case='cased', lang='en', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True) + >>> normalizer_en.normalize("") + # normalize list of entries + >>> normalizer_en.normalize_list(["", ""]) + # normalize .json manifest entries + >>> normalizer_en.normalize_manifest(manifest=, n_jobs=-1, batch_size=300, + output_filename=, text_field="text", + punct_pre_process=False, punct_post_process=False) + +""" + + +class Normalizer: + """ + Normalizer class that converts text from written to spoken form. + Useful for TTS preprocessing. + + Args: + input_case: Input text capitalization, set to 'cased' if text contains capital letters. + This flag affects normalization rules applied to the text. Note, `lower_cased` won't lower case input. + lang: language specifying the TN rules, by default: English + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + post_process: WFST-based post processing, e.g. to remove extra spaces added during TN. + Note: punct_post_process flag in normalize() supports all languages. + max_number_of_permutations_per_split: a maximum number + of permutations which can be generated from input sequence of tokens. + verbose: whether to print intermediate meta information + """ + + def __init__( + self, + input_case: str, + lang: str = 'en', + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + lm: bool = False, + post_process: bool = True, + max_number_of_permutations_per_split: int = 729, + ): + assert input_case in ["lower_cased", "cased"] + + self.post_processor = None + + if lang == "en": + from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.post_processing import PostProcessingFst + from zonos2.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + + if deterministic: + from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst + else: + if lm: + from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify_lm import ClassifyFst + else: + from zonos2.vendor.nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify_with_audio import ( + ClassifyFst, + ) + elif lang == 'ru': + # Ru TN only support non-deterministic cases and produces multiple normalization options + # use normalize_with_audio.py + from zonos2.vendor.nemo_text_processing.text_normalization.ru.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.ru.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'de': + from zonos2.vendor.nemo_text_processing.text_normalization.de.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.de.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'es': + from zonos2.vendor.nemo_text_processing.text_normalization.es.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.es.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'fr': + from zonos2.vendor.nemo_text_processing.text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.fr.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'sv': + from zonos2.vendor.nemo_text_processing.text_normalization.sv.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.sv.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'hu': + from zonos2.vendor.nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.hu.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'zh': + from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ar': + from zonos2.vendor.nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.ar.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'hi': + from zonos2.vendor.nemo_text_processing.text_normalization.hi.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.hi.verbalizers.post_processing import PostProcessingFst + from zonos2.vendor.nemo_text_processing.text_normalization.hi.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'it': + from zonos2.vendor.nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'hy': + from zonos2.vendor.nemo_text_processing.text_normalization.hy.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.hy.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'rw': + from zonos2.vendor.nemo_text_processing.text_normalization.rw.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.rw.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ja': + from zonos2.vendor.nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'vi': + from zonos2.vendor.nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.vi.verbalizers.post_processing import PostProcessingFst + from zonos2.vendor.nemo_text_processing.text_normalization.vi.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'pt': + from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ko': + from zonos2.vendor.nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst + from zonos2.vendor.nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + else: + raise NotImplementedError(f"Language {lang} has not been supported yet.") + + self.input_case = input_case + self.tagger = ClassifyFst( + input_case=self.input_case, + deterministic=deterministic, + cache_dir=cache_dir, + overwrite_cache=overwrite_cache, + whitelist=whitelist, + ) + + self.verbalizer = VerbalizeFinalFst( + deterministic=deterministic, cache_dir=cache_dir, overwrite_cache=overwrite_cache + ) + self.max_number_of_permutations_per_split = max_number_of_permutations_per_split + self.parser = TokenParser() + self.lang = lang + self.moses_detokenizer = MosesDetokenizer(lang=lang) + + def normalize_list( + self, + texts: List[str], + verbose: bool = False, + punct_pre_process: bool = False, + punct_post_process: bool = False, + batch_size: int = 1, + n_jobs: int = 1, + **kwargs, + ): + """ + NeMo text normalizer + + Args: + texts: list of input strings + verbose: whether to print intermediate meta information + punct_pre_process: whether to do punctuation pre-processing + punct_post_process: whether to do punctuation post-processing + n_jobs: the maximum number of concurrently running jobs. If -1 all CPUs are used. If 1 is given, + no parallel computing code is used at all, which is useful for debugging. For n_jobs below -1, + (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one are used. + batch_size: Number of examples for each process + + Returns converted list input strings + """ + + def _process_batch(batch, verbose, punct_pre_process, punct_post_process, **kwargs): + """ + Normalizes batch of text sequences + Args: + batch: list of texts + verbose: whether to print intermediate meta information + punct_pre_process: whether to do punctuation pre-processing + punct_post_process: whether to do punctuation post-processing + """ + normalized_lines = [ + self.normalize( + text, + verbose=verbose, + punct_pre_process=punct_pre_process, + punct_post_process=punct_post_process, + **kwargs, + ) + for text in tqdm(batch) + ] + return normalized_lines + + # to save intermediate results to a file + batch = min(len(texts), batch_size) + + try: + normalized_texts = Parallel(n_jobs=n_jobs)( + delayed(_process_batch)(texts[i : i + batch], verbose, punct_pre_process, punct_post_process, **kwargs) + for i in range(0, len(texts), batch) + ) + except BaseException as e: + raise e + + normalized_texts = list(itertools.chain(*normalized_texts)) + return normalized_texts + + def _estimate_number_of_permutations_in_nested_dict( + self, token_group: Dict[str, Union[OrderedDict, str, bool]] + ) -> int: + num_perms = 1 + for k, inner in token_group.items(): + if isinstance(inner, dict): + num_perms *= self._estimate_number_of_permutations_in_nested_dict(inner) + num_perms *= factorial(len(token_group)) + return num_perms + + def _split_tokens_to_reduce_number_of_permutations(self, tokens: List[dict]) -> List[List[dict]]: + """ + Splits a sequence of tokens in a smaller sequences of tokens in a way that maximum number of composite + tokens permutations does not exceed ``max_number_of_permutations_per_split``. + + For example, + + .. code-block:: python + + # setup normalizer with self.max_number_of_permutations_per_split=6 + tokens = [ + {"tokens": {"date": {"year": "twenty eighteen", "month": "december", "day": "thirty one"}}}, + {"tokens": {"date": {"year": "twenty eighteen", "month": "january", "day": "eight"}}}, + ] + split = normalizer._split_tokens_to_reduce_number_of_permutations(tokens) + assert split == [ + [{"tokens": {"date": {"year": "twenty eighteen", "month": "december", "day": "thirty one"}}}], + [{"tokens": {"date": {"year": "twenty eighteen", "month": "january", "day": "eight"}}}], + ] + + Date tokens contain 3 items each which gives 6 permutations for every date. Since there are 2 dates, total + number of permutations would be ``6 * 6 == 36``. Parameter ``self.max_number_of_permutations_per_split`` equals 6, + so input sequence of tokens is split into 2 smaller sequences. + + Args: + tokens: a list of dictionaries, possibly nested. + + Returns: + a list of smaller sequences of tokens resulting from ``tokens`` split. + """ + splits = [] + prev_end_of_split = 0 + current_number_of_permutations = 1 + for i, token_group in enumerate(tokens): + n = self._estimate_number_of_permutations_in_nested_dict(token_group) + if n * current_number_of_permutations > self.max_number_of_permutations_per_split: + splits.append(tokens[prev_end_of_split:i]) + prev_end_of_split = i + current_number_of_permutations = 1 + if n > self.max_number_of_permutations_per_split: + raise ValueError( + f"Could not split token list with respect to condition that every split can generate number of " + f"permutations less or equal to " + f"`self.max_number_of_permutations_per_split={self.max_number_of_permutations_per_split}`. " + f"There is an unsplittable token group that generates more than " + f"{self.max_number_of_permutations_per_split} permutations. Try to increase " + f"`--max_number_of_permutations_per_split` parameter." + ) + current_number_of_permutations *= n + splits.append(tokens[prev_end_of_split:]) + assert sum([len(s) for s in splits]) == len(tokens) + return splits + + def normalize( + self, text: str, verbose: bool = False, punct_pre_process: bool = False, punct_post_process: bool = False + ) -> str: + """ + Main function. Normalizes tokens from written to spoken form + e.g. 12 kg -> twelve kilograms + + Args: + text: string that may include semiotic classes + punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ] + punct_post_process: whether to normalize punctuation + verbose: whether to print intermediate meta information + + Returns: spoken form + """ + logger.setLevel('DEBUG' if verbose else 'INFO') + if len(text.split()) > 500: + logger.warning( + "Your input is too long and could take a long time to normalize. " + "Use split_text_into_sentences() to make the input shorter and then call normalize_list()." + ) + original_text = text + if punct_pre_process: + text = pre_process(text) + text = text.strip() + if not text: + logger.debug(text) + return text + text = pynini.escape(text) + tagged_lattice = self.find_tags(text) + tagged_text = Normalizer.select_tag(tagged_lattice) + logger.debug(tagged_text) + + self.parser(tagged_text) + tokens = self.parser.parse() + split_tokens = self._split_tokens_to_reduce_number_of_permutations(tokens) + output = "" + for s in split_tokens: + try: + tags_reordered = self.generate_permutations(s) + verbalizer_lattice = None + for tagged_text in tags_reordered: + tagged_text = pynini.escape(tagged_text) + + verbalizer_lattice = self.find_verbalizer(tagged_text) + if verbalizer_lattice.num_states() != 0: + break + if verbalizer_lattice is None: + logger.warning(f"No permutations were generated from tokens {s}") + return text + output += ' ' + Normalizer.select_verbalizer(verbalizer_lattice) + except Exception as e: + logger.warning("Failed text: " + text + str(e)) + return text + output = SPACE_DUP.sub(' ', output[1:]) + + if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: + output = self.post_process(output) + + if punct_post_process: + # do post-processing based on Moses detokenizer + output = self.moses_detokenizer.detokenize([output], unescape=False) + output = post_process_punct(input=original_text, normalized_text=output) + return output + + def normalize_line( + self, + line: str, + verbose: bool = False, + punct_pre_process=False, + punct_post_process=True, + text_field: str = "text", + output_field: str = "normalized", + **kwargs, + ): + """ + Normalizes "text_field" in line from a .json manifest + + Args: + line: line of a .json manifest + verbose: set to True to see intermediate output of normalization + punct_pre_process: set to True to do punctuation pre-processing + punct_post_process: set to True to do punctuation post-processing + text_field: name of the field in the manifest to normalize + output_field: name of the field in the manifest to save normalized text + """ + line = json.loads(line) + + normalized_text = self.normalize( + text=line[text_field], + verbose=verbose, + punct_pre_process=punct_pre_process, + punct_post_process=punct_post_process, + **kwargs, + ) + line[output_field] = normalized_text + return line + + def normalize_manifest( + self, + manifest: str, + n_jobs: int, + punct_pre_process: bool, + punct_post_process: bool, + batch_size: int, + output_filename: Optional[str] = None, + text_field: str = "text", + verbose: bool = False, + **kwargs, + ): + """ + Normalizes "text_field" from .json manifest. + + Args: + manifest: path to .json manifest file + n_jobs: the maximum number of concurrently running jobs. If -1 all CPUs are used. If 1 is given, + no parallel computing code is used at all, which is useful for debugging. For n_jobs below -1, + (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one are used. + punct_pre_process: set to True to do punctuation pre-processing + punct_post_process: set to True to do punctuation post-processing + batch_size: number of samples to process per iteration (int) + output_filename: path to .json file to save normalized text + text_field: name of the field in the manifest to normalize + **kwargs are need for audio-based normalization that requires extra args + """ + + def _process_batch( + batch_idx: int, + batch: List[str], + dir_name: str, + verbose=verbose, + punct_pre_process=False, + punct_post_process=True, + text_field: str = "text", + output_field: str = "normalized", + **kwargs, + ): + """ + Normalizes batch of text sequences + Args: + batch: list of texts + batch_idx: batch index + dir_name: path to output directory to save results + """ + normalized_lines = [ + self.normalize_line( + line=line, + verbose=verbose, + punct_post_process=punct_post_process, + punct_pre_process=punct_pre_process, + text_field=text_field, + output_field=output_field, + **kwargs, + ) + for line in tqdm(batch) + ] + + with open(f"{dir_name}/{batch_idx:06}.json", "w") as f_out: + for line in normalized_lines: + if isinstance(line[output_field], set): + if len(line[output_field]) > 1: + logger.warning("Len of " + str(line[output_field]) + " > 1 ") + line[output_field] = line[output_field].pop() + + f_out.write(json.dumps(line, ensure_ascii=False) + '\n') + + logger.info(f"Batch -- {batch_idx} -- is complete") + + if output_filename is None: + output_filename = manifest.replace('.json', '_normalized.json') + + with open(manifest, 'r') as f: + lines = f.readlines() + + logger.warning(f'Normalizing {len(lines)} line(s) of {manifest}...') + + # to save intermediate results to a file + batch = min(len(lines), batch_size) + + tmp_dir = "/tmp/parts" + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir) + + Parallel(n_jobs=n_jobs)( + delayed(_process_batch)( + idx, + lines[i : i + batch], + tmp_dir, + text_field=text_field, + verbose=verbose, + punct_pre_process=punct_pre_process, + punct_post_process=punct_post_process, + **kwargs, + ) + for idx, i in enumerate(range(0, len(lines), batch)) + ) + + # aggregate all intermediate files + with open(output_filename, "w") as f_out: + for batch_f in sorted(glob(f"{tmp_dir}/*.json")): + with open(batch_f, "r") as f_in: + lines = f_in.read() + f_out.write(lines) + + logger.warning(f'Normalized version saved at {output_filename}') + + def split_text_into_sentences(self, text: str, additional_split_symbols: str = "") -> List[str]: + r""" + Split text into sentences. + + Args: + text: text + additional_split_symbols: Symbols to split sentences if eos sentence split resulted in a long sequence. + Use '|' as a separator between symbols, for example: ';|:'. Use '\s' to split by space. + + Returns list of sentences + """ + lower_case_unicode = "" + upper_case_unicode = "" + + if self.lang == "ru": + lower_case_unicode = '\u0430-\u04ff' + upper_case_unicode = '\u0410-\u042f' + + # end of quoted speech - to be able to split sentences by full stop + text = re.sub(r"([\.\?\!])([\"\'])", r"\g<2>\g<1> ", text) + + # remove extra space + text = re.sub(r" +", " ", text) + + # remove space in the middle of the lower case abbreviation to avoid splitting into separate sentences + matches = re.findall(rf"[a-z{lower_case_unicode}]\.\s[a-z{lower_case_unicode}]\.", text) + for match in matches: + text = text.replace(match, match.replace(". ", ".")) + + # Read and split transcript by utterance (roughly, sentences) + split_pattern = rf"(? List[str]: + """ + Creates reorderings of dictionary elements and serializes as strings + + Args: + d: (nested) dictionary of key value pairs + + Return permutations of different string serializations of key value pairs + """ + l = [] + if PRESERVE_ORDER_KEY in d.keys(): + d_permutations = [d.items()] + else: + d_permutations = itertools.permutations(d.items()) + for perm in d_permutations: + subl = [""] + for k, v in perm: + if isinstance(v, str): + subl = ["".join(x) for x in itertools.product(subl, [f"{k}: \"{v}\" "])] + elif isinstance(v, OrderedDict): + rec = self._permute(v) + subl = ["".join(x) for x in itertools.product(subl, [f" {k} {{ "], rec, [f" }} "])] + elif isinstance(v, bool): + subl = ["".join(x) for x in itertools.product(subl, [f"{k}: true "])] + else: + raise ValueError("Key: " + str(k) + " Value: " + str(v)) + l.extend(subl) + return l + + def generate_permutations(self, tokens: List[dict]): + """ + Generates permutations of string serializations of list of dictionaries + + Args: + tokens: list of dictionaries + + Returns string serialization of list of dictionaries + """ + + def _helper(prefix: str, token_list: List[dict], idx: int): + """ + Generates permutations of string serializations of given dictionary + + Args: + token_list: list of dictionaries + prefix: prefix string + idx: index of next dictionary + + Returns string serialization of dictionary + """ + if idx == len(token_list): + yield prefix + return + token_options = self._permute(token_list[idx]) + for token_option in token_options: + yield from _helper(prefix + token_option, token_list, idx + 1) + + return _helper("", tokens, 0) + + def find_tags(self, text: str) -> 'pynini.FstLike': + """ + Given text use tagger Fst to tag text + + Args: + text: sentence + + Returns: tagged lattice + """ + lattice = text @ self.tagger.fst + return lattice + + @staticmethod + def select_tag(lattice: 'pynini.FstLike') -> str: + """ + Given tagged lattice return shortest path + + Args: + lattice: pynini.FstLike tag lattice + + Returns: shortest path + """ + tagged_text = pynini.shortestpath(lattice, nshortest=1, unique=True).string() + return tagged_text + + def find_verbalizer(self, tagged_text: str) -> 'pynini.FstLike': + """ + Given tagged text creates verbalization lattice + This is context-independent. + + Args: + tagged_text: input text + + Returns: verbalized lattice + """ + lattice = tagged_text @ self.verbalizer.fst + return lattice + + @staticmethod + def select_verbalizer(lattice: 'pynini.FstLike') -> str: + """ + Given verbalized lattice return shortest path + + Args: + lattice: verbalization lattice + text: full text line to raise in case of an exception + + Returns: shortest path + """ + output = pynini.shortestpath(lattice, nshortest=1, unique=True).string() + # lattice = output @ self.verbalizer.punct_graph + # output = pynini.shortestpath(lattice, nshortest=1, unique=True).string() + return output + + def post_process(self, normalized_text: 'pynini.FstLike') -> str: + """ + Runs post-processing graph on normalized text + + Args: + normalized_text: normalized text + + Returns: shortest path + """ + normalized_text = normalized_text.strip() + if not normalized_text: + return normalized_text + normalized_text = pynini.escape(normalized_text) + + if self.post_processor is not None: + normalized_text = top_rewrite(normalized_text, self.post_processor.fst) + return normalized_text + + +def parse_args(): + parser = ArgumentParser() + input = parser.add_mutually_exclusive_group() + input.add_argument("--text", dest="input_string", help="input string", type=str) + input.add_argument( + "--input_file", + dest="input_file", + help="input file path. " + "The input file can be either a .txt file containing one example for normalization per line or " + "a .json manifest file. Field to normalize in .json manifest is specified with `--manifest_text_field` arg.", + type=str, + ) + parser.add_argument( + '--manifest_text_field', + help="The field in a .json manifest to normalize (applicable only when input_file is a .json manifest)", + type=str, + default="text", + ) + parser.add_argument( + "--output_field", + help="Name of the field in a .json manifest in which to save normalized text (applicable only when input_file is a .json manifest)", + type=str, + default="normalized", + ) + parser.add_argument('--output_file', dest="output_file", help="Output file path", type=str) + parser.add_argument( + "--language", + help="language", + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + default="en", + type=str, + ) + parser.add_argument( + "--input_case", + help="Input text capitalization, set to 'cased' if text contains capital letters." + "This argument affects normalization rules applied to the text. Note, `lower_cased` won't lower case input.", + choices=["lower_cased", "cased"], + default="cased", + type=str, + ) + parser.add_argument("--verbose", help="print info for debugging", action='store_true') + parser.add_argument( + "--no_post_process", + help="WFST-based post processing, e.g. to remove extra spaces added during TN, normalize punctuation marks [could differ from the input]. Only Eng is supported, not supported in Sparrowhawk", + action="store_true", + ) + parser.add_argument( + "--punct_post_process", + help="Add this flag to enable punctuation post processing to match input.", + action="store_true", + ) + parser.add_argument( + "--punct_pre_process", + help="Add this flag to add spaces around square brackets, otherwise text between square brackets won't be normalized", + action="store_true", + ) + parser.add_argument("--overwrite_cache", help="Add this flag to re-create .far grammar files", action="store_true") + parser.add_argument( + "--whitelist", + help="Path to a file with with whitelist replacement," + "e.g., for English, whitelist files are stored under text_normalization/en/data/whitelist", + default=None, + type=str, + ) + parser.add_argument( + "--cache_dir", + help="path to a dir with .far grammar file. Set to None to avoid using cache", + default=None, + type=str, + ) + parser.add_argument("--n_jobs", default=-2, type=int, help="The maximum number of concurrently running jobs") + parser.add_argument("--batch_size", default=200, type=int, help="Number of examples for each process") + parser.add_argument( + "--max_number_of_permutations_per_split", + default=729, + type=int, + help="a maximum number of permutations which can be generated from input sequence of tokens.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + whitelist = os.path.abspath(args.whitelist) if args.whitelist else None + + if not args.input_string and not args.input_file: + raise ValueError("Either `--text` or `--input_file` required") + + normalizer = Normalizer( + input_case=args.input_case, + post_process=not args.no_post_process, + cache_dir=args.cache_dir, + overwrite_cache=args.overwrite_cache, + whitelist=whitelist, + lang=args.language, + max_number_of_permutations_per_split=args.max_number_of_permutations_per_split, + ) + start_time = perf_counter() + if args.input_string: + output = normalizer.normalize( + args.input_string, + verbose=args.verbose, + punct_pre_process=args.punct_pre_process, + punct_post_process=args.punct_post_process, + ) + print("=" * 40) + print(output) + print("=" * 40) + elif args.input_file: + if args.input_file.endswith(".json"): + normalizer.normalize_manifest( + args.input_file, + n_jobs=args.n_jobs, + punct_pre_process=args.punct_pre_process, + punct_post_process=args.punct_post_process, + batch_size=args.batch_size, + text_field=args.manifest_text_field, + output_field=args.output_field, + output_filename=args.output_file, + verbose=args.verbose, + ) + + else: + logger.info("Loading data: " + args.input_file) + data = load_file(args.input_file) + + logger.info("- Data: " + str(len(data)) + " sentences") + normalizer_prediction = normalizer.normalize_list( + data, + verbose=args.verbose, + punct_pre_process=args.punct_pre_process, + punct_post_process=args.punct_post_process, + ) + if args.output_file: + write_file(args.output_file, normalizer_prediction) + logger.info(f"- Normalized. Writing out to {args.output_file}") + else: + logger.info(normalizer_prediction) + + logger.info(f"Execution time: {perf_counter() - start_time:.02f} sec") diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/preprocessing_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/preprocessing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0cd7d04e6371866c0e38925c49cc1191be66de13 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/preprocessing_utils.py @@ -0,0 +1,70 @@ +from typing import List + + +def _split(sentences: List[str], delimiter: str, max_len: int, min_len: int): + """ + Splits sentences based by the specified delimiter. Will attempt to split and combine sentences to get target + min/max length. + + Args: + sentences: Sentences to split into segments. + delimiter: symbol to split by + max_len: the maximum number of symbols in the output sentences (the result will be the closest len match) + min_len: the minimum number of the output sentences (the result will be the closest len match) + """ + result = [] + for sent in sentences: + if len(sent) < max_len: + result.append(sent) + continue + + split_sent = sent.split(delimiter) + # keep the delimiter + split_sent = [(s + delimiter).strip() for s in split_sent[:-1]] + [split_sent[-1]] + + if "," in delimiter: + # split based on comma usually results in too short utterance, combine sentences + # that result in a single word split. It's usually not recommended to do that for other delimiters. + comb = [] + for s in split_sent: + # if the previous sentence is too short, combine it with the current sentence + if len(comb) > 0 and (len(comb[-1].split()) <= min_len or len(s.split()) <= min_len): + comb[-1] = comb[-1] + " " + s + else: + comb.append(s) + result.extend(comb) + else: + result.extend(split_sent) + return result + + +def additional_split(sentences: List[str], split_on_symbols: str, max_len: int = 1000, min_len: int = 2) -> List[str]: + """ + Splits sentences by split_on_symbols. + + Args: + sentences: Sentences to split into segments. + split_on_symbols: Symbols to split sentences if eos sentence split resulted in a long sequence. + Use '|' as a separator between symbols, for example: ';|:| ', will attempt to split each sentence + by semi-colon ";", colon ":", and space " ". + max_len: the maximum number of symbols in the output sentences (the result will be the closest len match) + min_len: the minimum number of the output sentences (the result will be the closest len match) + """ + if len(split_on_symbols) == 0: + return sentences + + split_on_symbols = split_on_symbols.split("|") + + another_sent_split = [] + for sent in sentences: + split_sent = [sent] + for delimiter in split_on_symbols: + if len(delimiter) == 0: + continue + split_sent = _split( + split_sent, delimiter + " " if delimiter != " " else delimiter, max_len=max_len, min_len=min_len + ) + another_sent_split.extend(split_sent) + + sentences = [s.strip() for s in another_sent_split if s.strip()] + return sentences diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd13e2d6abefaa0bbd5b9395f7fa5fb469ce61e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/months.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6713229bd54d3893037a3465874d1dc1a3f265aa --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/months.tsv @@ -0,0 +1,21 @@ +1 janeiro +01 janeiro +2 fevereiro +02 fevereiro +3 março +03 março +4 abril +04 abril +5 maio +05 maio +6 junho +06 junho +7 julho +07 julho +8 agosto +08 agosto +9 setembro +09 setembro +10 outubro +11 novembro +12 dezembro diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ee24567bc538c771135d091a7c335e2fb7103710 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv @@ -0,0 +1,3 @@ +/ +. +- diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d04c0fa503402e93b9f7cdaa46130933cfa6d38f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv @@ -0,0 +1 @@ +preposition de diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b9daa19a5b6fca1118279e4fc229899127725886 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv @@ -0,0 +1,7 @@ +.com ponto com +.com.br ponto com ponto br +.gov.br ponto gov ponto br +.org ponto org +.net ponto net +.edu ponto edu +.br ponto br diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..698bc277346eeb6c0ec83085e65e5eee41ff6246 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv @@ -0,0 +1,16 @@ +google google +usuario usuario +microsoft microsoft +amazon amazon +facebook facebook +meta meta +netflix netflix +spotify spotify +samsung samsung +apple apple +linkedin linkedin +instagram instagram +whatsapp whatsapp +oracle oracle +adobe adobe +paypal paypal diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..50b4eeb656e7e04cec72a9fd2dd0920eb2ef531c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv @@ -0,0 +1,6 @@ +gmail +nvidia +outlook +hotmail +yahoo +live diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..85c8a1b1057a8662dfd7b2cc050a2044a5de21e0 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. ponto +- traço +_ underscore +! exclamação +# cerquilha +$ dólar +% por cento +& e comercial +' apóstrofo +* asterisco ++ mais +/ barra += igual +? interrogação +^ acento circunflexo +` crase +{ chave esquerda +| barra vertical +} chave direita +~ til +, vírgula +: dois pontos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..22c9ed16cfa58101b1b1cee97491329b67f04192 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv @@ -0,0 +1,2 @@ +segundo meio +terceiro terço diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b19c443640f1e133b256283b5ffa0233912a05ea --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv @@ -0,0 +1 @@ +mil milésimo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7efeba5e659225612c1251627faf80ec745df01f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv @@ -0,0 +1,5 @@ +connector e +minus menos +plural_suffix s +avos_suffix avos +avos_between diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d9cbfb87ffa42b452c57f8b04165cf79bbeb7b8d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv @@ -0,0 +1,64 @@ +h horas +min minutos +s segundos +ms milissegundos +ns nanossegundos +μs microssegundos +t toneladas +kg quilos +kg quilogramas +g gramas +mg miligramas +μm micrômetros +nm nanômetros +mm milímetros +cm centímetros +cm² centímetros quadrados +cm³ centímetros cúbicos +m metros +m² metros quadrados +m³ metros cúbicos +km quilômetros +km² quilômetros quadrados +ha hectares +kph quilômetros por hora +mph milhas por hora +m/s metros por segundo +l litros +ml mililitros +kgf quilogramas força +% por cento +°F fahrenheit +°F graus fahrenheit +°C graus celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watts +kW quilowatts +MW megawatts +GW gigawatts +Wh watts hora +kWh quilowatts hora +MWh megawatts hora +GWh gigawatts hora +kV quilovolts +V volts +mV milivolts +A amperes +mA miliamperes +rpm rotações por minuto +db decibéis +cal calorias +kcal quilocalorias +G gramas +KG quilos +KG quilogramas +KM quilômetros +M metros +L litros +ML mililitros +M2 metros quadrados +M^2 metros quadrados +C graus celsius diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv new file mode 100644 index 0000000000000000000000000000000000000000..242aba14ef9c943c71eafada2a4795c23d2ef7c8 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv @@ -0,0 +1,65 @@ +h hora +min minuto +s segundo +ms milissegundo +ns nanossegundo +μs microssegundo +t tonelada +kg quilo +kg quilograma +g grama +mg miligrama +μm micrômetro +nm nanômetro +mm milímetro +cm centímetro +cm² centímetro quadrado +cm³ centímetro cúbico +m metro +m² metro quadrado +m³ metro cúbico +km quilômetro +km² quilômetro quadrado +ha hectare +kph quilômetro por hora +mph milha por hora +m/s metro por segundo +l litro +ml mililitro +kgf quilograma força +% por cento +°F fahrenheit +°C celsius +°F grau fahrenheit +°C grau celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watt +kW quilowatt +MW megawatt +GW gigawatt +Wh watt hora +kWh quilowatt hora +MWh megawatt hora +GWh gigawatt hora +kV quilovolt +V volt +mV milivolt +A ampere +mA miliampere +rpm rotação por minuto +db decibel +cal caloria +kcal quilocaloria +G grama +KG quilo +KG quilograma +KM quilômetro +M metro +L litro +ML mililitro +M2 metro quadrado +M^2 metro quadrado +C celsius diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b23d1bcce2d97fef90002d08caa8a9b4d64616b7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv @@ -0,0 +1,5 @@ +US$ dólar americano +R$ real +€ euro +£ libra esterlina +$ dólar diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv new file mode 100644 index 0000000000000000000000000000000000000000..feca270ff86feeaea6b3b51b64213b11e59a2488 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv @@ -0,0 +1,5 @@ +dólar americano dólares americanos +real reais +euro euros +libra esterlina libras esterlinas +dólar dólares diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4e9a95d660e38795c14d42bfe7b27826fd3ab1ab --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv @@ -0,0 +1,5 @@ +US$ centavo +R$ centavo +€ centavo +£ centavo +$ centavo diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2ac61dff7b9ad8d6e95565699d210689afc80b42 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv @@ -0,0 +1 @@ +centavo centavos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv new file mode 100644 index 0000000000000000000000000000000000000000..04ea91ee48988240de856e5040eb3723ca4c1f02 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv @@ -0,0 +1,4 @@ +connector e +thousand mil +hundred_100 cem +hundred_1 cento diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c84a95f5359a8eaff24f9ba7c7351e4757e8d5a6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv @@ -0,0 +1,3 @@ +001 mil e um +010 mil e dez +100 mil e cem diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f6257d9d1458bc9cce5ff4365069a41c288676d4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv @@ -0,0 +1,2 @@ +separator vírgula +minus menos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1859416c8aa891bfec2a223cec8ce1b71833535a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 um +2 dois +3 três +4 quatro +5 cinco +6 seis +7 sete +8 oito +9 nove diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..620f512b30a74bdbfab6b6b2d5528032c535992c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +2 duzentos +3 trezentos +4 quatrocentos +5 quinhentos +6 seiscentos +7 setecentos +8 oitocentos +9 novecentos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a94cbd5533e798f094fcc059ed1cf8034a6823f4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv @@ -0,0 +1,9 @@ +mil +milhão +milhões +bilhão +bilhões +trilhão +trilhões +quatrilhão +quatrilhões diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a3dffe4e02a17cadca32648cca15da5b772ba210 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv @@ -0,0 +1,4 @@ +one_label plural_suffix magnitude_zeros +um milhão milhões 0 +um bilhão bilhões 9 +um trilhão trilhões 12 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..50c4e0b8dffcdad174bd096a2be0bf94b38e5917 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv @@ -0,0 +1,10 @@ +10 dez +11 onze +12 doze +13 treze +14 catorze +15 quinze +16 dezesseis +17 dezessete +18 dezoito +19 dezenove diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..43c4a8bc6a6171bb273495444f5de9626fec754d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +2 vinte +3 trinta +4 quarenta +5 cinquenta +6 sessenta +7 setenta +8 oitenta +9 noventa diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..29be0f38bb1b695a302206225694cfdb5980e4c3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zero diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5fefbc3b8a485d72ab978226d38c4e58adbf3bcb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv @@ -0,0 +1,10 @@ +primeiro um +segundo dois +terceiro três +quarto quatro +quinto cinco +sexto seis +sétimo sete +oitavo oito +nono nove +décimo dez diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c75ae5ed053454d25cc51aca449e043586798c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv @@ -0,0 +1,11 @@ +primeiro primeira +segundo segunda +terceiro terceira +quarto quarta +quinto quinta +sexto sexta +sétimo sétima +oitavo oitava +nono nona +décimo décima +ésimo ésima diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6d919a86c8f4cbf1225141147c35a0c2c6c1e7ee --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv @@ -0,0 +1,10 @@ +centésimo cem +centésimo cento +ducentésimo duzentos +trecentésimo trezentos +quadringentésimo quatrocentos +quincentésimo quinhentos +sexcentésimo seiscentos +septingentésimo setecentos +octingentésimo oitocentos +noningentésimo novecentos diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb6933fe65980da5db95883ff9192c75121df8ad --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv @@ -0,0 +1,2 @@ +connector_in e +connector_out diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1b1e191c9162940a084de3dac378c8995bc91482 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv @@ -0,0 +1,9 @@ +décimo primeiro onze +décimo segundo doze +décimo terceiro treze +décimo quarto catorze +décimo quinto quinze +décimo sexto dezesseis +décimo sétimo dezessete +décimo oitavo dezoito +décimo nono dezenove diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f40700034097f41f90f4fae52a52aa822991fcc7 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv @@ -0,0 +1,8 @@ +vigésimo vinte +trigésimo trinta +quadragésimo quarenta +quinquagésimo cinquenta +sexagésimo sessenta +septuagésimo setenta +octogésimo oitenta +nonagésimo noventa diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7d59e35ebc24fe76607a77e4dc6cc269cb32b052 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv @@ -0,0 +1,2 @@ +ip ip +endereço de ip endereço de i p diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..62efeccd5b412d94324e25ded3e938fca79a2f0e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv @@ -0,0 +1,4 @@ +ligue para +telefone +celular +meu número é diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a942795f4f24495859696037b42792add0664891 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv @@ -0,0 +1,4 @@ +manhã da manhã 6 11 +tarde da tarde 12 17 +noite da noite 18 23 +madrugada da madrugada 0 5 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f5559c7116309e621659b6b4b1db91d91836abcf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv @@ -0,0 +1,521 @@ +a +aoj +aəj +aː +aːʲ +aː͡j +aː͡ɨ̯ +aˤ +aˤː +a̠ +a̠ː +a̰ +a͡e +a͡i +a͡iː +a͡i̯ +a͡j +a͡o +a͡u +a͡uː +a͡u̯ +a͡w +a͡ə +a͡ɨ̯ +a͡ɪ +a͡ʊ +b +bʱ +bʲ +bː +b̥ +c +cʰ +cː +ç +d +dʲ +dː +d̥ +d̪ +d̪ʱ +d͡z +d͡zʷ +d͡zː +d͡ʑ +d͡ʒ +d͡ʒʱ +d͡ʒʲ +d͡ʒː +e +eː +eːʲ +eː͡j +ẽː +ẽ͡j̃ +e̞ +e̞ː +e̯ +e͡i +e͡iː +e͡ɨ̯ +f +fʲ +fː +h +hː +i +iəj +iəw +iʲ +iː +iːʲ +ĩː +i̥ +i̯ +i͡u +i͡ə +i͡ɛ +j +jː +j̃ +k +kʰ +kʰː +kʲ +kʲʼ +kʷ +kʷʼ +kʼ +kː +k̚ +k̚ʲ +k̟̚ +k͈ +k͡p̚ +l +lʲ +lː +l̥ +l̩ +m +mʲ +mʲː +mː +m̥ +m̩ +n +nʲ +nː +n̥ +n̩ +o +oʲ +oː +oːʲ +ò +õ͡j̃ +õ͡w̃ +o̝ +o̞ +o̞ː +o̯ +o̰ +o͡u +o͡uː +p +pʰ +pʰː +pʲ +pʷʼ +pʼ +pː +p̚ +p̚ʲ +p͈ +p͜f +p͡f +q +qʷ +qʼ +r +rʲ +rː +r̂ +r̂ː +r̥ +r̩ +s +sʰ +sʲ +sʼ +sː +s͈ +t +tʰ +tʰː +tʲ +tʷʼ +tʼ +tː +t̚ +t̪ +t̪ʰ +t͈ +t͜s +t͡s +t͡sʰ +t͡sʰː +t͡sʲ +t͡sʷ +t͡sʼ +t͡sː +t͡ɕ +t͡ɕʰ +t͡ɕ͈ +t͡ʂ +t͡ʂʼ +t͡ʃ +t͡ʃʰ +t͡ʃʰː +t͡ʃʲ +t͡ʃʷ +t͡ʃʼ +t͡ʃː +u +uəj +uʲ +uː +uːʲ +ũː +ũ͡j̃ +u̯ +u͡e +u͡i +u͡j +u͡ɔ +u͡ə +v +vʲ +vː +w +w̃ +x +xʷ +xː +y +yː +yːʲ +y̯ +z +zʲ +zː +z̥ +à +àː +á +áː +â +âː +ã +ã̠ +æ +æː +æ̀ +æ̀ː +æ̂ +æ̂ː +æ͡ɪ +æ͡ʉ +ç +è +èː +é +éː +ê +êː +ì +ìː +í +íː +î +îː +ï +ð +ò +òː +ó +óː +ô +ôː +õ +õː +õ̞ +ø +øː +øːʲ +ø̯ +ù +ùː +ú +úː +û +ûː +ā +āː +ē +ēː +ĕ +ĕ͡ə +ě +ěː +ħ +ĩ +ĩː +ī +īː +ŋ +ŋʲ +ŋ̊ +ŋ̍ +ŋ̟ +ŋ̩ +ŋ͡m +ō +ŏ +ŏ͡ə +œ +œː +œ̃ +œ͡i +œ͡iː +œ͡ʏ +ř +řː +ũ +ũː +ū +ūː +ŭ +ŭ͡ə +ǎ +ǎː +ǐ +ǐː +ǒ +ǒː +ǔ +ǔː +ǣ +ǣː +ɐ +ɐː +ɐ̃ +ɐ̃͡j̃ +ɐ̃͡w̃ +ɐ̯ +ɐ̯̯ +ɑ +ɑː +ɑ̃ +ɑ̃ː +ɒ +ɒʲ +ɒː +ɓ +ɔ +ɔː +ɔˤː +ɔ̀ +ɔ̀ː +ɔ́ +ɔ́ː +ɔ̃ +ɔ̃ː +ɔ̰ +ɔ͡i̯ +ɔ͡ə +ɔ͡ɨ̯ +ɔ͡ɪ +ɔ͡ʊ +ɕ +ɕʰ +ɕː +ɕ͈ +ɖ +ɖʱ +ɗ +ɘ +ɘː +ə +əː +əˤ +ə̀ +ə́ +ə̃ +ə̯ +ə͡u̯ +ə͡w +ə͡ɨ +ə͡ɨ̯ +ɚ +ɛ +ɛʲ +ɛː +ɛˤː +ɛ̀ +ɛ̀ː +ɛ́ +ɛ́ː +ɛ̂ +ɛ̂ː +ɛ̃ +ɛ̃ː +ɛ̄ +ɛ̄ː +ɛ̰ +ɛ͡i +ɛ͡i̯ +ɛ͡u +ɛ͡u̯ +ɛ͡ɪ +ɛ͡ʊ +ɜ +ɜː +ɝ +ɝː +ɟ +ɟː +ɟ͡ʝ +ɡ +ɡʱ +ɡʲ +ɡʷ +ɡː +ɡ̊ +ɣ +ɤ +ɥ +ɦ +ɨ +ɨəj +ɨː +ɨ̃ᵝ +ɨ̞ +ɨ̥ᵝ +ɨ̯ +ɨ͡u̯ +ɨ͡w +ɨ͡ə +ɨᵝ +ɨᵝː +ɪ +ɪː +ɪ̀ +ɪ́ +ɪ̃ +ɪ̯ +ɪ̰ +ɪ͡u̯ +ɪ͡ʊ +ɫ +ɫː +ɬ +ɬʼ +ɭ +ɮ +ɯ +ɯː +ɯ̟̃ᵝ +ɯ̟̊ᵝ +ɯ̟ᵝ +ɯ̟ᵝː +ɰ +ɰ̃ +ɰᵝ +ɱ +ɱ̩ +ɲ +ɲː +ɲ̊ +ɲ̟ +ɳ +ɴ +ɸ +ɸʷ +ɹ +ɻ +ɽ +ɽʱ +ɾ +ɾʲ +ɾː +ɾ̝̊ +ʀ +ʁ +ʁʷ +ʁː +ʂ +ʂʷ +ʃ +ʃʰ +ʃʲ +ʃʷ +ʃʷʼ +ʃʼ +ʃː +ʈ +ʈʰ +ʉ +ʉː +ʊ +ʊ̀ +ʊ́ +ʊ̃ +ʊ̯ +ʊ̯͡i +ʊ̯͡ɨ +ʊ̰ +ʋ +ʌ +ʌ̹ +ʍ +ʎ +ʏ +ʏː +ʏ̯ +ʐ +ʐʷ +ʑ +ʒ +ʒʲ +ʒʷ +ʒː +ʔ +ʔʲ +ʔʷ +ʝ +˦ˀ˥ +˦˥ +˦˧˥ +˦˩ +˧ˀ˨ +˧˦ +˧˧ +˧˨ +˧˩ +˨˩ +˨˩˦ +˨˩˨ +β +θ +χ +χʷ +χː +ḛ +ḭ +ṵ +ẽ +ẽː +ẽ̞ +‿ \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7f7b525e314a55f3542bb139544dc642e882cb6e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv @@ -0,0 +1,23 @@ +& e comercial +# cerquilha +@ arroba +§ parágrafo +™ marca comercial +® marca registrada +© direitos autorais +_ sublinhado +% por cento +* asterisco ++ mais +/ barra += igual +^ acento circunflexo +| barra vertical +~ til +$ dólar +£ libra esterlina +€ euro +₩ won +¥ ienes +° grau +º ordinal masculino diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/tts.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/pt/data/whitelist/tts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8588fb62559b07d46a64c578b5765d9ba51e5721 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/graph_utils.py @@ -0,0 +1,193 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Portuguese (PT) text normalization graph utilities. + +Self-contained module with no dependency on en.graph_utils. Provides character/digit +symbols (NEMO_*), space helpers (delete_space, insert_space, delete_extra_space), +GraphFst base class, generator_main for FAR export, and PT-specific helpers +(filter_cardinal_punctuation, shift_cardinal_gender_pt). +""" + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +# ---- Character/digit symbols (same semantics as EN) ---- +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_SPACE = " " +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, pynini.accep('"')).optimize() +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() + +MIN_NEG_WEIGHT = -0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ").optimize() + +delete_preserve_order = pynini.closure(pynutil.delete(" preserve_order: true")) + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]) -> None: + """ + Export one or more graphs to an OpenFst Finite State Archive (FAR) file. + + Args: + file_name: path to the output .far file. + graphs: mapping of rule names to FST graphs to export. + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +class GraphFst: + """ + Base class for all Portuguese text normalization grammar FSTs. + + Args: + name: name of the grammar (e.g. "cardinal", "decimal"). + kind: either "classify" or "verbalize". + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization). + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(os.path.abspath(__file__)) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + +# ---- PT-specific (Brazilian: 1.000.000 or 1 000 000) ---- +cardinal_separator = pynini.string_map([".", " "]) + + +def filter_cardinal_punctuation(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Parse digit groups separated by cardinal_separator (e.g. 1.000.000) then apply fst. + + Args: + fst: FST that maps digit string to verbalized cardinal. + + Returns: + Composed FST that accepts digit strings with optional thousand separators. + """ + exactly_three = NEMO_DIGIT**3 + up_to_three = pynini.closure(NEMO_DIGIT, 1, 3) + cardinal_string = pynini.closure(NEMO_DIGIT, 1) + cardinal_string |= ( + up_to_three + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three + pynutil.delete(cardinal_separator)) + + exactly_three + ) + return cardinal_string @ fst + + +def shift_cardinal_gender_pt(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Apply Portuguese masculine-to-feminine conversion for cardinal strings, e.g. + "um" -> "uma", "dois" -> "duas", "duzentos" -> "duzentas". + + Args: + fst: FST producing masculine cardinal verbalization. + + Returns: + FST that produces feminine form when composed with the same input. + """ + fem_ones = pynini.cdrewrite( + pynini.cross("um", "uma"), + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + fem_twos = pynini.cdrewrite( + pynini.cross("dois", "duas"), + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + fem_hundreds = pynini.cdrewrite( + pynini.cross("entos", "entas"), + pynini.union("duz", "trez", "quatroc", "quinh", "seisc", "setec", "oitoc", "novec"), + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + return fst @ fem_ones @ fem_twos @ fem_hundreds + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..7d6052818cdeee91a2f85d9e7abe636014d9efad --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/cardinal.py @@ -0,0 +1,303 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from functools import reduce + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + filter_cardinal_punctuation, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese cardinals, e.g. + "1000" -> cardinal { integer: "mil" } + "2.000.000" -> cardinal { integer: "dois milhões" } + "-5" -> cardinal { negative: "true" integer: "cinco" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + specials = { + row[0]: row[1] for row in load_labels(get_abs_path("data/numbers/cardinal_specials.tsv")) if len(row) >= 2 + } + connector_e = insert_space + pynutil.insert(specials["connector"]) + insert_space + thousand = specials["thousand"] + hundred_100 = specials["hundred_100"] + hundred_1 = specials["hundred_1"] + + scale_rows = load_labels(get_abs_path("data/numbers/scales.tsv")) + scales = [(row[0], row[1], int(row[2])) for row in scale_rows if len(row) >= 3 and row[2].strip().isdigit()] + + _num = lambda p: pynini.string_file(get_abs_path(f"data/numbers/{p}")) + zero, digit, teens, tens, hundreds = ( + _num("zero.tsv"), + _num("digit.tsv"), + _num("teens.tsv"), + _num("tens.tsv"), + _num("hundreds.tsv"), + ) + digits_no_one = (NEMO_DIGIT - "1") @ digit + + graph_tens = teens | (tens + (pynutil.delete("0") | (connector_e + digit))) + self.tens = graph_tens.optimize() + self.two_digit_non_zero = pynini.union(digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + digit)).optimize() + + # After "X00" hundreds (oitocentos, …), suffix "01"-"09" needs leading zero stripped + # (graph_tens has no path for "09"; connector+digit only consumes one digit). + graph_hundreds = hundreds + pynini.union( + pynutil.delete("00"), + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + (connector_e + digit), + ) + # "100" -> cem only (cross("1", cento)+delete("00") would also match "100" but + # yields "cento"; OpenFst vs pynini top_rewrite can disagree on ties — Sparrowhawk). + graph_hundreds |= pynini.cross("100", hundred_100) + graph_hundreds |= pynini.cross("1", hundred_1) + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + ) + self.hundreds = graph_hundreds.optimize() + + h_comp_base = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + h_comp = h_comp_base | (pynutil.delete("00") + digit) + h_comp_no_one = h_comp_base | (pynutil.delete("00") + digits_no_one) + + pure_tens_input = pynini.union(*[pynini.accep(str(d * 10)) for d in range(1, 10)]) + graph_pure_tens_only = pure_tens_input @ graph_tens + graph_compound_tens = (pynini.closure(NEMO_DIGIT, 2, 2) - pure_tens_input) @ graph_tens + + graph_pure_components = pynini.union( + pynutil.delete("0") + graph_pure_tens_only, + pynutil.delete("00") + digit, + hundreds + pynutil.delete("00"), + pynini.cross("100", hundred_100), + ) + graph_compound_hundreds = pynini.union( + pynini.cross("1", hundred_1) + + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + ), + hundreds + + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + (connector_e + digit), + ), + ) + + suffix_after_mil = pynini.union( + pynutil.delete("000"), + (connector_e + graph_pure_components), + (insert_space + graph_compound_hundreds), + # Use connector_e so "2024" -> dois mil e vinte e quatro (not dois mil vinte e quatro). + (connector_e + pynutil.delete("0") + graph_compound_tens), + ) + + t_comp = pynini.union( + pynutil.delete("000") + h_comp, + h_comp_no_one + insert_space + pynutil.insert(thousand) + suffix_after_mil, + pynini.cross("001", thousand) + suffix_after_mil, + ) + t_comp_no_one = pynini.union( + pynutil.delete("000") + h_comp_no_one, + h_comp_no_one + + insert_space + + pynutil.insert(thousand) + + ((insert_space + h_comp) | pynutil.delete("000")), + pynini.cross("001", thousand) + ((insert_space + h_comp) | pynutil.delete("000")), + ) + + graph_large_scales = pynini.accep("") + for one_label, plural_suffix, _ in reversed(scales): + g = pynutil.add_weight(pynini.cross("000001", one_label), -0.001) + g |= t_comp_no_one + pynutil.insert(plural_suffix) + g |= pynutil.delete("000000") + g += insert_space + graph_large_scales += g + + # 9/12-digit: scale block + trailing (million+thousands, billion+9digits) + scale_3_mil = self._scale_block_3(scales[0][0], scales[0][1], h_comp_no_one) + scale_3_bi = self._scale_block_3(scales[1][0], scales[1][1], h_comp_no_one) + graph_9 = self._build_scale_trailing_graph(scale_3_mil, t_comp, 6, 9) + graph_12 = self._build_scale_trailing_graph(scale_3_bi, graph_9, 9, 12) + pure_9, pure_12 = self._pure_inputs(9), self._pure_inputs(12) + trail_9 = (pure_9 @ graph_9, (NEMO_DIGIT**9 - pure_9) @ graph_9) + trail_12 = (pure_12 @ graph_12, (NEMO_DIGIT**12 - pure_12) @ graph_12) + + # Units 6 (u6): pure get "e" after scale; compound no "e" + u6_one = pynini.cross("000001", "1") @ digit + u6_pure = pynini.union( + u6_one, + pynini.cross("001000", thousand), + pynini.cross("000010", "10") @ graph_tens, + pynini.cross("000100", hundred_100), + (pynini.cross("010000", "10") @ graph_tens) + insert_space + pynutil.insert(thousand), + pynini.cross("100000", hundred_100) + insert_space + pynutil.insert(thousand), + ) + u6_compound = (NEMO_DIGIT**6 - self._pure_inputs(6)) @ t_comp + u6 = u6_pure | u6_compound + z18 = pynini.accep("0" * 18) # 18 zeros: branch no "e" + smaller_e = (connector_e + u6_pure) | u6_compound | pynutil.delete("0" * 6) + smaller = u6 | pynutil.delete("0" * 6) + graph_24 = (((NEMO_DIGIT**18 - z18) + NEMO_DIGIT**6) @ (graph_large_scales + smaller_e)) | ( + (z18 + NEMO_DIGIT**6) @ (pynutil.delete(z18) + smaller) + ) + + trail_by_z = {9: trail_9, 12: trail_12} + magnitude_patterns = [ + self._build_magnitude_pattern( + one_label, + plural_suffix, + magnitude_zeros, + trail_by_z.get(magnitude_zeros), + connector_e, + insert_space, + digit, + graph_tens, + graph_hundreds, + ) + for one_label, plural_suffix, magnitude_zeros in scales + if magnitude_zeros > 0 + ] + + pad = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0) + pad = pad @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) @ NEMO_DIGIT**24 + norm = pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite( + delete_space, "", "[EOS]", NEMO_SIGMA + ) + norm = norm @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + self.graph = reduce(lambda a, b: a | b, magnitude_patterns, pad @ graph_24 @ norm) | zero + self.graph = filter_cardinal_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def _scale_block_3(self, one_label, plural_suffix, component_no_one): + """001->one_label, 000->'', else component+plural.""" + return pynini.union( + pynini.cross("001", one_label), + pynini.cross("000", ""), + (NEMO_DIGIT**3 - pynini.accep("001") - pynini.accep("000")) + @ (component_no_one + insert_space + pynutil.insert(plural_suffix)), + ) + + def _build_scale_trailing_graph(self, scale_3, sub_graph, trailing_len, total_len): + """total_len digits = scale_3 + trailing; no trailing space when trailing all zeros.""" + zt, ztotal = "0" * trailing_len, "0" * total_len + scale_nonzero = NEMO_DIGIT**3 - pynini.accep("000") + branches = [ + (pynini.accep("000") + NEMO_DIGIT**trailing_len) @ (pynutil.delete("000") + sub_graph), + (scale_nonzero + (NEMO_DIGIT**trailing_len - pynini.accep(zt))) @ (scale_3 + insert_space + sub_graph), + (scale_nonzero + pynini.accep(zt)) @ (scale_3 + pynutil.delete(zt)), + (pynini.accep("000") + pynini.accep(zt)) @ pynutil.delete(ztotal), + ] + return pynini.union(*branches) + + @staticmethod + def _pure_inputs(num_digits): + """Inputs 1, 10, 100, ... as num_digits-digit strings.""" + return pynini.union(*[pynini.accep(str(10**k).zfill(num_digits)) for k in range(0, num_digits)]) + + def _magnitude_graph( + self, + one_word, + plural_suffix, + zero_count, + graph_digit, + graph_tens, + graph_hundreds, + connector_e, + insert_space, + trailing_pair=None, + ): + """Round (1–3 digit + scale + zeros); optional trailing (e + pure | space + compound).""" + zeros = "0" * zero_count + round_pats = [] + trail_pats = [] if trailing_pair else None + for L in (1, 2, 3): + total = zero_count + L + if L == 1: + lead = pynini.cross("1", one_word) | ((NEMO_DIGIT - "1") @ graph_digit + pynutil.insert(plural_suffix)) + else: + lead = pynini.closure(NEMO_DIGIT, L, L) @ (graph_tens if L == 2 else graph_hundreds) + pynutil.insert( + plural_suffix + ) + lead_fst = NEMO_DIGIT**L @ lead + round_pats.append(pynini.closure(NEMO_DIGIT, total, total) @ (lead_fst + pynutil.delete(zeros))) + if trailing_pair: + pure, compound = trailing_pair + trail_part = NEMO_DIGIT**zero_count @ (connector_e + pure) | NEMO_DIGIT**zero_count @ ( + insert_space + compound + ) + trail_pats.append(pynini.closure(NEMO_DIGIT, total, total) @ (lead_fst + trail_part)) + graph_round = pynini.union(*round_pats) + graph_trail = pynini.union(*trail_pats) if trail_pats else None + return graph_round, graph_trail + + def _build_magnitude_pattern( + self, + one_label, + plural_suffix, + magnitude_zeros, + trailing_pair, + connector_e, + insert_space, + graph_digit, + graph_tens, + graph_hundreds, + ): + """Restrict length; round + optional non-zero trailing.""" + restrict = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, magnitude_zeros, magnitude_zeros + 2) + graph_round, graph_trail = self._magnitude_graph( + one_label, + plural_suffix, + magnitude_zeros, + graph_digit, + graph_tens, + graph_hundreds, + connector_e, + insert_space, + trailing_pair, + ) + if graph_trail is None: + return pynutil.add_weight(restrict @ graph_round, -1.0) + non_zero_trail = pynini.union( + *[NEMO_DIGIT**n + (NEMO_DIGIT**magnitude_zeros - pynini.accep("0" * magnitude_zeros)) for n in (1, 2, 3)] + ) + return pynutil.add_weight(restrict @ (graph_round | (non_zero_trail @ graph_trail)), -1.0) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..da7631616b69976ba586117117a73f92d0b4af01 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/date.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying Portuguese (Brazilian) dates, e.g. + 15/03/2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 15 de março de 2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 2024-03-15 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 03/15/2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + numbers = cardinal.graph + + month_rows = load_labels(get_abs_path("data/date/months.tsv")) + month_pairs = [(r[0], r[1]) for r in month_rows if len(r) >= 2] + month_to_word = pynini.string_map(month_pairs).optimize() + + day_10_31 = ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ pynini.union(*[str(x) for x in range(10, 32)]) @ numbers + day_02_09 = pynutil.delete("0") + (pynini.union(*[str(x) for x in range(2, 10)]) @ numbers) + day_2_9 = pynini.union(*[str(x) for x in range(2, 10)]) @ numbers + day_inner = pynini.union( + pynini.cross("01", "primeiro"), + day_10_31, + day_02_09, + day_2_9, + pynini.cross("1", "primeiro"), + ).optimize() + day_part = pynutil.insert('day: "') + day_inner + pynutil.insert('"') + + month_digits = ( + pynini.union("10", "11", "12") + | pynutil.delete("0") + pynini.union(*[str(x) for x in range(1, 10)]) + | pynini.union(*[str(x) for x in range(1, 10)]) + ) + month_num = month_digits @ month_to_word + month_part = pynutil.insert('month: "') + month_num + pynutil.insert('"') + + year_num = ((NEMO_DIGIT - "0") + NEMO_DIGIT**3) @ numbers + year_part = pynutil.insert('year: "') + year_num + pynutil.insert('"') + + preserve = pynutil.insert(" preserve_order: true") + + delete_de = delete_space + pynutil.delete("de") + delete_space + month_names = sorted({r[1] for r in month_rows if len(r) >= 2}, key=len, reverse=True) + text_pairs = [] + for name in month_names: + text_pairs.append((name, name)) + if name and name[0].islower(): + text_pairs.append((name[0].upper() + name[1:], name)) + month_written = pynutil.insert('month: "') + pynini.string_map(text_pairs).optimize() + pynutil.insert('"') + graph_text = day_part + delete_de + month_written + delete_de + year_part + preserve + + sep_path = get_abs_path("data/date/numeric_separators.tsv") + separators = [r[0].strip() for r in load_labels(sep_path) if r and r[0].strip()] + + one_or_two_digits = pynini.closure(NEMO_DIGIT, 1, 2) + year_four = (NEMO_DIGIT - "0") + NEMO_DIGIT**3 + _mdy_weight = 0.05 + + months_spoken = sorted({r[1] for r in month_rows if len(r) >= 2}) + day_spokens = set() + for n in range(1, 32): + for key in (str(n), f"{n:02d}"): + dstr = pynini.shortestpath(pynini.compose(pynini.accep(key), day_inner.optimize())).string() + day_spokens.add(dstr) + + _preserve_tail = " preserve_order: true" + + ymd_to_dmy_graph = None + mdy_to_dmy_graph = None + for month in months_spoken: + for day in day_spokens: + # After year: + sigma (year value + quotes), delete month/day and trailing preserve + # so the input is fully consumed (mdy_to_dmy does not need this: sigma eats the tail). + ymd_curr = ( + pynutil.insert('day: "' + day + '" month: "' + month + '" ') + + pynini.accep("year:") + + NEMO_SIGMA + + pynutil.delete(' month: "' + month + '" day: "' + day + '"' + _preserve_tail) + ) + ymd_to_dmy_graph = ymd_curr if ymd_to_dmy_graph is None else pynini.union(ymd_to_dmy_graph, ymd_curr) + + mdy_curr = ( + pynutil.insert('day: "' + day + '" month: "' + month + '" ') + + pynutil.delete('month: "' + month + '" day: "' + day + '" ') + + pynini.accep("year:") + + NEMO_SIGMA + ) + mdy_to_dmy_graph = mdy_curr if mdy_to_dmy_graph is None else pynini.union(mdy_to_dmy_graph, mdy_curr) + + ymd_to_dmy_graph = ymd_to_dmy_graph.optimize() + mdy_to_dmy_graph = mdy_to_dmy_graph.optimize() + + patterns = [graph_text] + for sep in separators: + sep_accep = pynini.accep(pynini.escape(sep)) + del_sep = pynutil.delete(sep_accep) + + dmy_core = day_part + del_sep + insert_space + month_part + del_sep + insert_space + year_part + preserve + iso_core = year_part + del_sep + insert_space + month_part + del_sep + insert_space + day_part + preserve + mdy_core = month_part + del_sep + insert_space + day_part + del_sep + insert_space + year_part + preserve + + lhs_dmy = one_or_two_digits + sep_accep + one_or_two_digits + sep_accep + year_four + lhs_iso = year_four + sep_accep + one_or_two_digits + sep_accep + one_or_two_digits + lhs_mdy = one_or_two_digits + sep_accep + one_or_two_digits + sep_accep + year_four + + patterns.append(pynini.compose(lhs_dmy, dmy_core)) + patterns.append( + pynutil.add_weight( + pynini.compose( + pynini.compose(lhs_mdy, mdy_core), + mdy_to_dmy_graph, + ), + _mdy_weight, + ) + ) + patterns.append( + pynini.compose( + pynini.compose(lhs_iso, iso_core), + ymd_to_dmy_graph, + ) + ) + + self.fst = self.add_tokens(pynini.union(*patterns).optimize()).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..908c5a0f764f006e28a50e2c9cf7d284cea344f5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/decimal.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese decimal numbers, e.g. + "1,26" -> decimal { integer_part: "um" fractional_part: "dois seis" } + "0,01" -> decimal { integer_part: "zero" fractional_part: "zero um" } + "-1,26" -> decimal { negative: "true" ... } + "1,33 milhões" / "1 milhão" -> decimal { ... quantity: "milhões" / "milhão" } + + The fractional mantissa (after the comma) is always read digit-by-digit (0–9), + including leading zeros. Integer part and quantities still use cardinals. + + Args: + cardinal: CardinalFst instance for integer verbalization in tags. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + _num = lambda name: pynini.string_file(get_abs_path(f"data/numbers/{name}")).optimize() + + comma = pynutil.delete(",") + quantity_words = _num("quantity_words.tsv") + digit = _num("digit.tsv") + zero = _num("zero.tsv") + graph_digit_or_zero = pynini.union(digit, zero) + digit_by_digit = (graph_digit_or_zero + pynini.closure(insert_space + graph_digit_or_zero)).optimize() + + fractional_digits = pynini.closure(NEMO_DIGIT, 1, 15) + graph_fractional = ( + pynutil.insert('fractional_part: "') + (fractional_digits @ digit_by_digit) + pynutil.insert('"') + ) + + non_zero_lead = pynini.difference(NEMO_DIGIT, pynini.accep("0")) + + graph_integer_zero = ( + pynutil.insert('integer_part: "') + pynini.cross("0", "zero") + pynutil.insert('"') + insert_space + ) + decimal_when_zero = graph_integer_zero + comma + insert_space + graph_fractional + + graph_integer_pos = ( + pynutil.insert('integer_part: "') + + (non_zero_lead + pynini.closure(NEMO_DIGIT, 0, 11)) @ cardinal_graph + + pynutil.insert('"') + + insert_space + ) + decimal_when_pos = graph_integer_pos + comma + insert_space + graph_fractional + + decimal_core = pynini.union(decimal_when_zero, decimal_when_pos) + integer_quantity = ( + pynutil.insert('integer_part: "') + + (pynini.closure(NEMO_DIGIT, 1, 12) @ cardinal_graph) + + pynutil.insert('"') + + insert_space + + delete_space + + pynutil.insert('quantity: "') + + quantity_words + + pynutil.insert('"') + ) + decimal_quantity = ( + decimal_core + delete_space + pynutil.insert('quantity: "') + quantity_words + pynutil.insert('"') + ) + final_graph_wo_sign = pynini.union(decimal_core, integer_quantity, decimal_quantity) + self.final_graph_wo_negative = final_graph_wo_sign.optimize() + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + final_graph = optional_minus + final_graph_wo_sign + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..57f8ccb14e1f70354244bcbf092a48074ee01381 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/electronic.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_ALPHA, NEMO_DIGIT, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic strings in pt-BR: + abc@hotmail.com -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + https://www.abc.com -> electronic { protocol: "https://www." domain: "abc.com" preserve_order: true } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + full_stop = pynini.accep(".") + at_symbol = "@" + protocol_string = "protocol" + domain_string = "domain" + username_string = "username" + http = "http" + https = "https" + www = "www" + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_full_stop = pynini.difference(symbols, full_stop) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_full_stop), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + domain_component = full_stop + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + ': "') + + (accepted_characters + pynini.closure(domain_component, 1)) + + pynutil.insert('"') + ) + + username = ( + pynutil.insert(username_string + ': "') + + all_characters + + pynutil.insert('"') + + pynini.cross(at_symbol, NEMO_SPACE) + ) + email = username + domain_graph + + social_tag = ( + pynini.cross(at_symbol, "") + + pynutil.insert(username_string + ': "') + + (accepted_characters | (accepted_characters + pynini.closure(domain_component, 1))) + + pynutil.insert('"') + ) + + protocol_start = pynini.accep(https + "://") | pynini.accep(http + "://") + protocol_end = pynini.accep(www + ".") + if not deterministic: + protocol_end |= pynini.cross(www + ".", "dáblio dáblio dáblio.") + + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert(protocol_string + ': "') + protocol + pynutil.insert('"') + url = protocol + pynutil.insert(NEMO_SPACE) + domain_graph + + graph = url | domain_graph | email | social_tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..313310a0c4064cca955ecf9fb247301462a12dc5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/fraction.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying Portuguese fraction numbers, e.g. + "1/2" -> fraction { numerator: "um" denominator: "meio" morphosyntactic_features: "ordinal" } + "2 3/4" -> fraction { integer_part: "dois" numerator: "três" denominator: "quarto" ... } + "2/11" -> fraction { numerator: "dois" denominator: "onze" morphosyntactic_features: "avos" } + + Args: + cardinal: CardinalFst instance for number parts. + ordinal: OrdinalFst instance for denominator 2-10 and exceptions. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + # Denominators 2–10 use ordinal form (no data file: fixed set) + two_to_ten = pynini.union(*[pynini.accep(str(d)) for d in range(2, 11)]).optimize() + + ord_digit_rows = load_labels(get_abs_path("data/ordinals/digit.tsv")) + ordinal_digit = pynini.string_map([(r[1], r[0]) for r in ord_digit_rows if len(r) >= 2]).optimize() + + ord_exc_rows = load_labels(get_abs_path("data/fractions/ordinal_exceptions.tsv")) + ordinal_exceptions = pynini.string_map([(r[0], r[1]) for r in ord_exc_rows if len(r) >= 2]).optimize() + + ord_hundreds_rows = load_labels(get_abs_path("data/ordinals/hundreds.tsv")) + ordinal_hundreds = pynini.string_map([(r[1], r[0]) for r in ord_hundreds_rows if len(r) >= 2]).optimize() + + powers_rows = load_labels(get_abs_path("data/fractions/powers_of_ten.tsv")) + powers_of_ten = pynini.string_map([(r[0], r[1]) for r in powers_rows if len(r) >= 2]).optimize() + + denom_ordinal_form = two_to_ten @ cardinal_graph @ ordinal_digit + denom_ordinal_form = denom_ordinal_form @ pynini.cdrewrite(ordinal_exceptions, "", "", NEMO_SIGMA) + denom_ordinal = ( + pynutil.insert('denominator: "') + + denom_ordinal_form + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + + denom_100 = ( + pynutil.insert('denominator: "') + + (pynini.accep("100") @ cardinal_graph @ ordinal_hundreds) + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + denom_1000 = ( + pynutil.insert('denominator: "') + + (pynini.accep("1000") @ cardinal_graph @ powers_of_ten) + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + + denom_ordinal_2_10_100_1000 = pynini.union(denom_ordinal, denom_100, denom_1000) + digit_plus = pynini.closure(NEMO_DIGIT, 1) + denom_avos_input = pynini.difference( + digit_plus, + pynini.union( + two_to_ten, + pynini.accep("100"), + pynini.accep("1000"), + ), + ) + denom_avos = ( + pynutil.insert('denominator: "') + + (denom_avos_input @ cardinal_graph) + + pynutil.insert('" morphosyntactic_features: "avos"') + ) + + denominator = pynini.union(denom_ordinal_2_10_100_1000, denom_avos) + + # Slash variants: ASCII /, Unicode ⁄ (U+2044), ∕ (U+2215); with or without spaces + slash_or_space_slash = pynini.union( + pynini.cross("/", '" '), + pynini.cross(" / ", '" '), + pynini.cross("\u2044", '" '), # fraction slash ⁄ + pynini.cross(" \u2044 ", '" '), + pynini.cross("\u2215", '" '), # division slash ∕ + pynini.cross(" \u2215 ", '" '), + ) + numerator = pynutil.insert('numerator: "') + cardinal_graph + slash_or_space_slash + fraction_core = numerator + denominator + + integer_part = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + insert_space + + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + mixed = integer_part + pynini.closure(NEMO_WHITE_SPACE, 1) + fraction_core + graph = optional_minus + pynini.union(mixed, fraction_core) + + self.fst = self.add_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..eb3ac1dbfe6e7cc8441cd532f682e2a9fce6dda5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/measure.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure (pt-BR), e.g. + 200 g -> measure { cardinal { integer: "duzentos" } units: "gramas" } + 1 kg -> measure { cardinal { integer: "um" } units: "quilo" } + 2,4 g -> measure { decimal { ... } units: "gramas" } + 1/2 l -> measure { fraction { ... } units: "litros" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_singular = pynini.string_file(get_abs_path("data/measure/measurements_singular.tsv")) + unit_plural = pynini.string_file(get_abs_path("data/measure/measurements_plural.tsv")) + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(unit_plural) + + optional_graph_negative = pynini.closure(pynini.accep("-"), 0, 1) + + unit_plural = pynutil.insert('units: "') + graph_unit_plural + pynutil.insert('"') + unit_singular_graph = pynutil.insert('units: "') + graph_unit_singular + pynutil.insert('"') + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(NEMO_SPACE, 0, 1) + unit_plural + + subgraph_cardinal = ( + (optional_graph_negative + (NEMO_SIGMA - "1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + unit_plural + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + unit_singular_graph + ) + + subgraph_fraction = fraction.fst + insert_space + pynini.closure(delete_space, 0, 1) + unit_plural + + final_graph = subgraph_decimal | subgraph_cardinal | subgraph_fraction + self.fst = self.add_tokens(final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..b91199d39afb16b7ce0f3e02ba54669fb30fe334 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/money.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money (pt-BR), e.g. + R$ 12 -> money { currency_maj: "reais" integer_part: "doze" } + R$ 12,05 -> money { currency_maj: "reais" integer_part: "doze" fractional_part: "cinco" + currency_min: "centavos" preserve_order: true } + R$ 0,20 -> money { fractional_part: "vinte" currency_min: "centavos" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + decimal_separator = pynini.accep(",") + maj_singular = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + maj_plural_map = pynini.string_file(get_abs_path("data/money/currency_major_plural.tsv")) + maj_plural_graph = maj_singular @ maj_plural_map + min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) + min_plural_map = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) + min_plural_graph = min_singular @ min_plural_map + + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + graph_maj_singular = pynutil.insert('currency_maj: "') + maj_singular + pynutil.insert('"') + graph_maj_plural = pynutil.insert('currency_maj: "') + maj_plural_graph + pynutil.insert('"') + + graph_integer_one = ( + pynutil.insert('integer_part: "') + (pynini.accep("1") @ cardinal_graph) + pynutil.insert('"') + ) + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + graph_decimal_plural = ((NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_plural + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"') + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert('currency_min: "') + min_singular + pynutil.insert('"') + graph_min_plural = pynutil.insert('currency_min: "') + min_plural_graph + pynutil.insert('"') + + maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert('fractional_part: "') + + (two_digits_fractional_part @ pynini.cross("1", "um")) + + pynutil.insert('"') + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_non_zero + ) + graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"') + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + if not deterministic: + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert('fractional_part: "') + + two_digits_fractional_part @ cardinal.two_digit_non_zero + + pynutil.insert('"'), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..23e5a1907f08fc3eddcf37f83deaff977e8e1ebb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/ordinal.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese ordinals, e.g. + "1º" / "1ª" -> ordinal { integer: "primeiro" / "primeira" morphosyntactic_features: "gender_masc" / "gender_fem" } + "21º" -> ordinal { integer: "vigésimo primeiro" morphosyntactic_features: "gender_masc" } + + Args: + cardinal: CardinalFst instance for composing compound ordinals. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + spec_rows = load_labels(get_abs_path("data/ordinals/specials.tsv")) + spec = {r[0]: r[1] for r in spec_rows if len(r) >= 2} + conn_in = spec.get("connector_in", " e ") + conn_out = spec.get("connector_out", NEMO_SPACE) + conn = pynini.cross(conn_in, conn_out) + + # Data: ordinal \t cardinal → FST cardinal→ordinal via load_labels + digit_rows = load_labels(get_abs_path("data/ordinals/digit.tsv")) + graph_digit = pynini.string_map([(r[1], r[0]) for r in digit_rows if len(r) >= 2]).optimize() + teen_rows = load_labels(get_abs_path("data/ordinals/teen.tsv")) + graph_teens = pynini.string_map([(r[1], r[0]) for r in teen_rows if len(r) >= 2]).optimize() + ties_rows = load_labels(get_abs_path("data/ordinals/ties.tsv")) + graph_ties = pynini.string_map([(r[1], r[0]) for r in ties_rows if len(r) >= 2]).optimize() + hundreds_rows = load_labels(get_abs_path("data/ordinals/hundreds.tsv")) + graph_hundreds = pynini.string_map([(r[1], r[0]) for r in hundreds_rows if len(r) >= 2]).optimize() + + graph_tens = pynini.union( + graph_teens, + graph_ties + pynini.closure(conn + graph_digit, 0, 1), + ) + graph_hundred_component = pynini.union( + graph_hundreds + pynini.closure(conn + pynini.union(graph_tens, graph_digit), 0, 1), + graph_tens, + graph_digit, + ) + ordinal_rewrite = graph_hundred_component.optimize() + ordinal_inner = cardinal_graph @ ordinal_rewrite + + opt_dot = pynini.closure(pynutil.delete("."), 0, 1) + suffix_masc = opt_dot + pynutil.delete(pynini.union("º", "°")) + suffix_fem = opt_dot + pynutil.delete("ª") + digit_block = pynini.closure(NEMO_DIGIT, 1, 3) + + to_ordinal_masc = (digit_block + suffix_masc) @ ordinal_inner + to_ordinal_fem = (digit_block + suffix_fem) @ ordinal_inner + + graph_masc = ( + pynutil.insert('integer: "') + + to_ordinal_masc + + pynutil.insert('" morphosyntactic_features: "gender_masc"') + ) + graph_fem = ( + pynutil.insert('integer: "') + to_ordinal_fem + pynutil.insert('" morphosyntactic_features: "gender_fem"') + ) + self.fst = self.add_tokens(pynini.union(graph_masc, graph_fem)).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/punctuation.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..437c59b21eba4adbc7df4f3e814c3d71c0439f7d --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/punctuation.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation (pt-BR pipeline). + Mirrors the English punctuation tagger; whitelist symbols are excluded via data/whitelist/symbol.tsv. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + [r"\[", r"\]"] + + symbol_path = get_abs_path("data/whitelist/symbol.tsv") + whitelist_rows = load_labels(symbol_path) if os.path.isfile(symbol_path) else [] + whitelist_symbols = [x[0] for x in whitelist_rows if x] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..1d92fcd0332f5a2b359b31fcb5ab810816732272 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/telephone.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_SPACE, NEMO_WHITE_SPACE, GraphFst, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying pt-BR telephone and IP formats, e.g. + (11) 99999-8888 -> telephone { number_part: "um um nove nove nove nove nove oito oito oito oito" } + +55 11 3333-4444 -> telephone { country_code: "mais cinco cinco" number_part: "um um três três três três quatro quatro quatro quatro" } + 192.168.1.1 -> telephone { number_part: "um nove dois ponto um seis oito ponto um ponto um" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize() + single_digits = (digit | zero).optimize() + + # Only strip grouping punctuation between digit blocks; do not delete spaces + # (deleting spaces would glue spoken digit words together in the output). + separators = pynini.union("-", ".") + # Optional separator after country / prompt (still allow +55 11 …). + delete_optional_sep = pynini.closure(pynutil.delete(separators), 0, 1) + # Require an explicit separator between local digit blocks so plain long cardinals + # (e.g. 3022110709) are not misclassified as telephone. + delete_required_separator = pynutil.delete(separators) + delete_optional_spaces = pynini.closure(pynutil.delete(NEMO_WHITE_SPACE), 0) + + def n_digits(n: int): + return pynini.closure(single_digits + insert_space, n - 1, n - 1) + single_digits + + country_digits = n_digits(1) | n_digits(2) | n_digits(3) + country_code = pynini.cross("+", "mais ") + country_digits + + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + tel_prompt_sequence = telephone_prompts + NEMO_SPACE + pynini.closure(country_code, 0, 1) + + country_code_graph = ( + pynutil.insert('country_code: "') + + (country_code | ip_prompts | tel_prompt_sequence) + + delete_optional_sep + + pynutil.insert('"') + ) + + area_code = (pynutil.delete("(") + n_digits(2) + pynutil.delete(")")) | n_digits(2) + + eleven_digit_graph = ( + area_code + + delete_optional_spaces + + insert_space + + n_digits(5) + + delete_required_separator + + insert_space + + n_digits(4) + ) + ten_digit_graph = ( + area_code + + delete_optional_spaces + + insert_space + + n_digits(4) + + delete_required_separator + + insert_space + + n_digits(4) + ) + nine_digit_graph = n_digits(5) + delete_required_separator + insert_space + n_digits(4) + eight_digit_graph = n_digits(4) + delete_required_separator + insert_space + n_digits(4) + seven_digit_graph = n_digits(3) + delete_required_separator + insert_space + n_digits(4) + + digit_to_str_graph = single_digits + pynini.closure(pynutil.insert(" ") + single_digits, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " ponto ") + digit_to_str_graph) ** 3 + + number_part = ( + eleven_digit_graph + | ten_digit_graph + | nine_digit_graph + | eight_digit_graph + | seven_digit_graph + | pynutil.add_weight(ip_graph, 0.01) + ) + number_part = pynutil.insert('number_part: "') + number_part + pynutil.insert('"') + + # "ramal" -> spoken "ramal …"; "extensão" / "ext." -> spoken "extensão …" (not "ext." letter-by-letter). + ext_core = n_digits(1) + pynini.closure(insert_space + n_digits(1), 0, 3) + extension_intro = delete_optional_spaces + ( + (pynutil.delete("ramal") + delete_optional_spaces + pynutil.insert("ramal ")) + | ( + (pynutil.delete("extensão") | pynutil.delete("ext.")) + + delete_optional_spaces + + pynutil.insert("extensão ") + ) + ) + ext_graph = pynutil.insert('extension: "') + extension_intro + ext_core + pynutil.insert('"') + + graph = ( + pynini.closure(country_code_graph + delete_optional_spaces + insert_space, 0, 1) + + number_part + + pynini.closure(delete_optional_spaces + insert_space + ext_graph, 0, 1) + ) + + self.fst = self.add_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..52d5a4aef02291612555b24e34d3cc29779f0784 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/time.py @@ -0,0 +1,180 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying Portuguese (Brazilian) time, e.g. + 14:30 -> time { hours: "catorze" minutes: "trinta" preserve_order: true } + 14:30:05 -> time { hours: "catorze" minutes: "trinta" seconds: "cinco" preserve_order: true } + 09:00:31 -> time { hours: "nove" minutes: "zero" seconds: "trinta e um" preserve_order: true } + 12:00 -> time { hours: "doze" preserve_order: true } + 11:00 da manhã -> time { hours: "onze" suffix: "da manhã" preserve_order: true } + 16:00 da tarde -> time { hours: "quatro" suffix: "da tarde" preserve_order: true } + 23:18 da tarde -> time { hours: "vinte e três" ... suffix: "da tarde" preserve_order: true } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph.optimize() + + hour_words = [] + for h in range(24): + key = str(h) + comp = pynini.compose(pynini.accep(key), cardinal_graph).optimize() + hour_words.append(pynini.shortestpath(comp, nshortest=1, unique=True).string()) + + hour_delete_fsts = [] + for h in range(24): + if h < 10: + hour_delete_fsts.append(pynini.union(pynutil.delete(str(h)), pynutil.delete(f"0{h}")).optimize()) + else: + hour_delete_fsts.append(pynutil.delete(str(h))) + + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal_graph + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal_graph + final_graph_minute = ( + pynutil.insert('minutes: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + + final_graph_second = ( + pynutil.insert('seconds: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + + # HMS verbalizer always expects ``minutes`` and ``seconds`` tags; bare ``delete("00")`` omits them. + zero_word = hour_words[0] + minutes_zero = ( + pynutil.delete("00") + pynutil.insert('minutes: "') + pynutil.insert(zero_word) + pynutil.insert('"') + ) + seconds_zero = ( + pynutil.delete("00") + pynutil.insert('seconds: "') + pynutil.insert(zero_word) + pynutil.insert('"') + ) + + delete_h = pynini.union( + pynutil.delete(pynini.accep(pynini.escape("h"))), + pynutil.delete(pynini.accep(pynini.escape("H"))), + ) + + time_delim = pynini.union( + pynini.accep(pynini.escape(":")), + pynini.accep(pynini.escape(".")), + ) + + period_rows = load_labels(get_abs_path("data/time/day_period_suffix.tsv")) + period_meta = [] + for row in period_rows: + if len(row) < 2 or not row[0].strip(): + continue + tail, tag_val = row[0].strip(), row[1].strip() + if len(row) < 4 or not row[2].strip().isdigit() or not row[3].strip().isdigit(): + raise ValueError( + f"day_period_suffix.tsv row must have 4 columns (tail, tag, hour_min, hour_max): {row!r}" + ) + h0, h1 = int(row[2].strip()), int(row[3].strip()) + allowed = frozenset(range(h0, h1 + 1)) + suf_fst = insert_space + delete_space + pynutil.delete("da") + delete_space + pynutil.delete(tail) + period_meta.append((tag_val, allowed, suf_fst, tail)) + + preserve = pynutil.insert(" preserve_order: true") + + mid_hm = pynutil.delete(time_delim) + (pynutil.delete("00") | insert_space + final_graph_minute) + mid_h_minute = delete_h + (pynutil.delete("00") | insert_space + final_graph_minute) + mid_h_only = delete_h + mid_hms = ( + pynutil.delete(time_delim) + + (minutes_zero | insert_space + final_graph_minute) + + pynutil.delete(time_delim) + + (seconds_zero | insert_space + final_graph_second) + ) + + graph_chunks = [] + for mid_after_hour in (mid_hm, mid_h_minute, mid_h_only, mid_hms): + branches = [] + for h in range(24): + hd = hour_delete_fsts[h] + hw24 = hour_words[h] + hour_tok_24 = pynutil.insert('hours: "') + pynutil.insert(hw24) + pynutil.insert('"') + branches.append(hd + hour_tok_24 + mid_after_hour + preserve) + for tag_val, allowed, suf, tail in period_meta: + keep_suffix, hour_idx = TimeFst._resolve_suffix_hour(h, tail, allowed) + hw_suf = hour_words[hour_idx] + hour_tok_suf = pynutil.insert('hours: "') + pynutil.insert(hw_suf) + pynutil.insert('"') + if keep_suffix: + branches.append( + hd + + hour_tok_suf + + mid_after_hour + + suf + + pynutil.insert(f' suffix: "{tag_val}"') + + preserve + ) + else: + # User wrote a period: always emit ``suffix:`` so TN does not drop it from speech + # (hours stay 24h when the period does not match the clock policy). + branches.append( + hd + + hour_tok_24 + + mid_after_hour + + suf + + pynutil.insert(f' suffix: "{tag_val}"') + + preserve + ) + graph_chunks.append(pynini.union(*branches).optimize()) + + final_graph = pynini.union(*graph_chunks).optimize() + self.fst = self.add_tokens(final_graph).optimize() + + @staticmethod + def _resolve_suffix_hour(h: int, period_tail: str, allowed: frozenset) -> tuple[bool, int]: + """Return (keep_suffix, hour_index) for ``hour_words[hour_index]`` when a day-period applies.""" + if period_tail == "manhã": + allowed_m = allowed | frozenset({1, 2, 3, 4, 5}) + if h not in allowed_m: + return False, h + return True, h + if period_tail == "tarde": + if h in allowed: + return True, 12 if h == 12 else h - 12 + if 1 <= h <= 5 and (h + 12) in allowed: + return True, h + return False, h + if period_tail == "noite": + if h in allowed: + return True, h - 12 + if 6 <= h <= 11 and (h + 12) in allowed: + return True, h + return False, h + if period_tail == "madrugada": + if h in allowed: + return True, h + return False, h + return False, h diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..d0f97a008a5244fdad086b21a68ea5da28e76d1b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py @@ -0,0 +1,135 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.taggers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all Portuguese classification grammars. This class can process an entire sentence (lower cased). + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files. + whitelist: path to a file with whitelist replacements. + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_pt_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + # Initialize Portuguese taggers + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal, ordinal, deterministic=deterministic) + decimal = DecimalFst(cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) + date = DateFst(cardinal, deterministic=deterministic) + time = TimeFst(cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + + punctuation = PunctuationFst(deterministic=deterministic) + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + + classify = ( + pynutil.add_weight(whitelist.fst, 1.01) + | pynutil.add_weight(date.fst, 1.1) + | pynutil.add_weight(time.fst, 1.1) + | pynutil.add_weight(measure.fst, 1.1) + | pynutil.add_weight(fraction.fst, 1.1) + | pynutil.add_weight(decimal.fst, 1.1) + | pynutil.add_weight(ordinal.fst, 1.1) + | pynutil.add_weight(cardinal.fst, 1.1) + | pynutil.add_weight(money.fst, 1.1) + | pynutil.add_weight(telephone.fst, 1.11) + | pynutil.add_weight(electronic.fst, 1.11) + | pynutil.add_weight(word_graph, 100) + ) + + # Wrap tokens properly + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + punct_graph = ( + pynutil.insert("tokens { ") + pynutil.add_weight(punctuation.fst, weight=2.1) + pynutil.insert(" }") + ) + + # Simple graph structure + graph = token + pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + token + ) + + # Allow punctuation + graph |= punct_graph + + self.fst = delete_space + graph + delete_space + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") + + +if __name__ == "__main__": + ClassifyFst(input_case="cased", deterministic=False) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..454a25dd1b487c9721e2fdc58e83554facf05152 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/whitelist.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import augment_labels_with_punct_at_end +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_SIGMA, + NEMO_UPPER, + GraphFst, + convert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +def _empty_fst() -> "pynini.FstLike": + """FST that accepts nothing (no whitelist rows).""" + return pynini.intersect(pynini.accep("a"), pynini.accep("b")).optimize() + + +def get_formats(input_f, input_case=INPUT_CASED, is_default=True): + """Abbreviation format variants (same idea as EN whitelist).""" + multiple_formats = load_labels(input_f) + if not multiple_formats: + return _empty_fst() + additional_options = [] + for x, y in multiple_formats: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((f"{x}.", y)) + additional_options.append((f"{x[0].upper() + x[1:]}", f"{y[0].upper() + y[1:]}")) + additional_options.append((f"{x[0].upper() + x[1:]}.", f"{y[0].upper() + y[1:]}")) + multiple_formats.extend(additional_options) + + if not is_default: + multiple_formats = [(x, f"|raw_start|{x}|raw_end||norm_start|{y}|norm_end|") for (x, y) in multiple_formats] + + return pynini.string_map(multiple_formats) + + +class WhiteListFst(GraphFst): + """ + Whitelist classifier for pt-BR TN. Data lives under pt/data/whitelist/ (may be empty). + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): + whitelist = load_labels(file) if os.path.isfile(file) else [] + if not whitelist: + return _empty_fst() + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y in whitelist] + else: + whitelist = [[x, y] for x, y in whitelist] + + if keep_punct_add_end: + whitelist.extend(augment_labels_with_punct_at_end(whitelist)) + + return pynini.string_map(whitelist) + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/tts.tsv")) + + symbol_path = get_abs_path("data/whitelist/symbol.tsv") + if os.path.isfile(symbol_path) and load_labels(symbol_path): + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + _get_whitelist_graph(input_case, symbol_path), + ).optimize() + + for x in [".", ". "]: + graph |= ( + NEMO_UPPER + + pynini.closure(pynutil.delete(x) + NEMO_UPPER, 2) + + pynini.closure(pynutil.delete("."), 0, 1) + ) + + if not deterministic: + alt_path = get_abs_path("data/whitelist/alternatives.tsv") + if os.path.isfile(alt_path) and load_labels(alt_path): + graph |= _get_whitelist_graph(input_case, alt_path, keep_punct_add_end=True) + fmt_path = get_abs_path("data/whitelist/alternatives_all_format.tsv") + if os.path.isfile(fmt_path) and load_labels(fmt_path): + graph |= get_formats(fmt_path, input_case=input_case) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..df9084227b02e055d3496f312f70beddc9fbe1df --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/taggers/word.py @@ -0,0 +1,81 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying words (pt-BR pipeline). + Same structure as the English word tagger; uses PT data paths and the passed PunctuationFst. + """ + + def __init__(self, punctuation: GraphFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + punct = punctuation.graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | NEMO_DIGIT).optimize() + graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + phoneme_unit = pynini.closure(NEMO_ALPHA, 1) + pynini.closure(NEMO_DIGIT) + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.accep(pynini.escape("]")) + ) + + punct_marks = pynini.union(*punctuation.punct_marks).optimize() + stress = pynini.union("ˈ", "'", "ˌ") + ipa_phoneme_unit = pynini.string_file(get_abs_path("data/whitelist/ipa_symbols.tsv")) + ipa_phonemes = ( + pynini.closure(stress, 0, 1) + + pynini.closure(ipa_phoneme_unit, 1) + + pynini.closure(stress | ipa_phoneme_unit) + ) + delim = (punct_marks | pynini.accep(" ")) ** (1, ...) + ipa_phonemes = ipa_phonemes + pynini.closure(delim + ipa_phonemes) + pynini.closure(delim, 0, 1) + ipa_phonemes = (pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]"))).optimize() + + if not deterministic: + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.accep(pynini.escape("]")) + ).optimize() + ipa_phonemes = ( + pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]")) + ).optimize() + + phoneme |= ipa_phonemes + self.graph = plurals._priority_union(convert_space(phoneme.optimize()), graph, NEMO_SIGMA) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..da4be3f8918f416fc741903b402ac4ba870a53c9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/utils.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Portuguese (PT) text normalization utilities. + +Provides get_abs_path for resolving data paths and load_labels for reading TSV label files. +""" +import csv +import os + + +def get_abs_path(rel_path: str) -> str: + """ + Resolve a path relative to this module to an absolute path. + + Args: + rel_path: path relative to the PT text normalization data directory. + + Returns: + Absolute path string. + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path: str): + """ + Load a TSV file as a list of rows (list of lists). + + Args: + abs_path: absolute path to a UTF-8 TSV file. + + Returns: + List of rows, each row a list of fields (e.g. from csv.reader). + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3fb699d9f6b1df3430f755849be91d63679cdc --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..cb0680b079e8bd23418dff5993a20fd238c9195e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + insert_space, + shift_cardinal_gender_pt, +) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese cardinal numbers, e.g. + cardinal { integer: "dois" } -> dois + cardinal { integer: "dois" } -> duas (feminine context via shift_cardinal_gender_pt) + cardinal { negative: "true" integer: "cinco" } -> menos cinco + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "menos") + insert_space, 0, 1) + self.optional_sign = optional_sign + + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + + # Generate masculine form (default) + graph_masc = optional_sign + integer + + # Generate feminine form using Portuguese gender conversion + graph_fem = shift_cardinal_gender_pt(graph_masc) + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + # Default to masculine for standalone numbers + # Context-aware gender selection will be handled by higher-level components + graph = graph_masc + + if not deterministic: + # For alternate renderings and contractions + # Portuguese doesn't have apocope like Spanish, but may have contractions + pass + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..aa239cd61e40d98c5e143ab290f2410f94e7e545 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/date.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese (Brazilian) dates, e.g. + date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + -> quinze de março de dois mil e vinte e quatro + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + vrows = load_labels(get_abs_path("data/date/verbal_phrases.tsv")) + vp = {r[0].strip(): r[1].strip() for r in vrows if len(r) >= 2 and r[0].strip()} + prep_word = vp.get("preposition", "de") + + quoted = pynini.closure(NEMO_NOT_QUOTE, 1) + + day_expr = pynutil.delete('day: "') + quoted + pynutil.delete('"') + month_expr = pynutil.delete('month: "') + quoted + pynutil.delete('"') + year_expr = pynutil.delete('year: "') + quoted + pynutil.delete('"') + + ws = delete_space + insert_space + glue = ws + pynutil.insert(prep_word) + insert_space + ws + + graph_dmy = day_expr + glue + month_expr + glue + year_expr + delete_preserve_order + self.fst = self.delete_tokens(graph_dmy).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..58f943b7b624b82d0b802f3e8a9e73e11e6d7b22 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese decimal numbers, e.g. + decimal { integer_part: "um" fractional_part: "vinte e seis" } -> um vírgula vinte e seis + decimal { negative: "true" integer_part: "um" ... } -> menos um vírgula ... + decimal { integer_part: "um" quantity: "milhão" } -> um milhão + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + labels = load_labels(get_abs_path("data/numbers/decimal_specials.tsv")) + spec = {r[0]: r[1] for r in labels if len(r) >= 2} + sep = spec.get("separator", "vírgula") + minus = spec.get("minus", "menos") + + optional_sign = pynini.closure(pynini.cross('negative: "true" ', minus) + insert_space, 0, 1) + + integer = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + fractional = pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + quantity = ( + delete_space + + insert_space + + pynutil.delete('quantity: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + integer_quantity = integer + quantity + decimal_part = ( + integer + + delete_space + + insert_space + + pynutil.insert(sep) + + insert_space + + fractional + + pynini.closure(quantity, 0, 1) + ) + + graph = optional_sign + pynini.union(integer_quantity, decimal_part) + + self.numbers = graph.optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..66c369bd8bfefeaeb29322de585ae99786c550c4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + +digit_no_zero = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +spoken_unit = pynini.string_file(get_abs_path("data/electronic/electronic_spoken_unit.tsv")) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + graph_digit = digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + # Prefer whole tokens (server names, TLDs, company/common words) over letter-by-letter. + user_segment = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common | spoken_unit + user_name = ( + pynutil.delete('username: "') + + (user_segment + pynini.closure(pynutil.insert(NEMO_SPACE) + user_segment)) + + pynutil.delete('"') + ) + user_name @= verbalize_characters + + convert_defaults = ( + pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common | spoken_unit + ) + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + domain = pynutil.delete('domain: "') + domain + pynutil.delete('"') + + protocol = ( + pynutil.delete('protocol: "') + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete('"') + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("arroba" + NEMO_SPACE) + domain + | (pynutil.insert("arroba" + NEMO_SPACE) + user_name) + ) + + self.fst = self.delete_tokens(self.graph + delete_preserve_order).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..e9a07908c2259571365dc4e928194ca5e8bb3353 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py @@ -0,0 +1,109 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese fraction numbers, e.g. + fraction { numerator: "um" denominator: "meio" morphosyntactic_features: "ordinal" } -> um meio + fraction { integer_part: "dois" numerator: "três" denominator: "quarto" } -> dois e três quartos + fraction { numerator: "dois" denominator: "onze" morphosyntactic_features: "avos" } -> dois onze avos + + Denominator routing (set by the tagger, unchanged here): + + * **Ordinal** (``morphosyntactic_features: "ordinal"``): denominators 2–10, 100, 1000 — spoken as + ordinals with plural ``s`` on the denominator when the numerator is not ``um`` (``três quartos``). + * **Avos** (``… "avos"``): all other positive integer denominators — cardinal denominator + the word + ``avos`` (``três onze avos``). Optional ``sobre`` between numerator and denominator is controlled by + ``data/fractions/specials.tsv`` key ``avos_between`` (empty = single space; ``sobre`` = ``… sobre …``). + * **Mixed** numbers use ``connector`` from the same TSV (default `` e ``) after the integer part. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + labels = load_labels(get_abs_path("data/fractions/specials.tsv")) + spec = {r[0]: r[1] for r in labels if len(r) >= 2} + connector_raw = spec.get("connector", " e ").strip() + connector = insert_space + pynutil.insert(connector_raw) + insert_space + minus = spec.get("minus", "menos ").rstrip() + plural_suffix = spec.get("plural_suffix", "s") + avos_word = spec.get("avos_suffix", " avos").strip() + avos_between = spec.get("avos_between", "").strip() + numerator_one_val = spec.get("numerator_one", "um") + denominator_half_val = spec.get("denominator_half", "meio") + + optional_sign = pynini.closure(pynini.cross('negative: "true" ', minus) + insert_space, 0, 1) + + integer = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('" ') + + numerator_one = pynutil.delete('numerator: "') + pynini.accep(numerator_one_val) + pynutil.delete('" ') + numerator_rest = ( + pynutil.delete('numerator: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE), pynini.accep(numerator_one_val)) + + pynutil.delete('" ') + ) + + denom_ordinal = ( + pynutil.delete('denominator: "') + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denom_meio = ( + pynutil.delete('denominator: "') + + pynini.accep(denominator_half_val) + + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denom_avos = ( + pynutil.delete('denominator: "') + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete('" morphosyntactic_features: "avos"') + ) + + fraction_ordinal_singular = numerator_one + insert_space + denom_ordinal + fraction_ordinal_plural = numerator_rest + insert_space + denom_ordinal + pynutil.insert(plural_suffix) + fraction_ordinal = pynini.union(fraction_ordinal_singular, fraction_ordinal_plural) + + if avos_between: + avos_mid = insert_space + pynutil.insert(avos_between) + insert_space + else: + avos_mid = insert_space + + fraction_avos = ( + pynini.union(numerator_one, numerator_rest) + + avos_mid + + denom_avos + + insert_space + + pynutil.insert(avos_word) + ) + + fraction = pynini.union(fraction_ordinal, fraction_avos) + mixed_um_meio = integer + connector + pynutil.delete('numerator: "' + numerator_one_val + '" " ') + denom_meio + optional_integer = pynini.closure(integer + connector + insert_space, 0, 1) + graph = optional_sign + pynini.union( + pynutil.add_weight(mixed_um_meio, -0.01), + optional_integer + fraction, + ) + + self.inner_graph = graph.optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..ad98ef35df51801b9e6750bc0f7ea19ddb1c71ec --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/measure.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_WHITE_SPACE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure (pt-BR), e.g. + measure { cardinal { integer: "duzentos" } units: "gramas" } -> duzentos gramas + measure { cardinal { integer: "um" } units: "hora" } -> uma hora + + Args: + decimal: DecimalFst verbalizer + cardinal: CardinalFst verbalizer + fraction: FractionFst verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + hours_unit = ( + pynutil.delete('units: "') + + pynini.union(pynini.accep("hora"), pynini.accep("horas")) + + pynutil.delete('"') + ) + non_hours_unit = ( + pynutil.delete('units: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("hora", "horas")) + + pynutil.delete('"') + ) + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + cardinal.graph_fem + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + hours_unit + ) + graph_cardinal |= ( + pynutil.delete("cardinal {") + + delete_space + + cardinal.graph_masc + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + non_hours_unit + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + (hours_unit | non_hours_unit) + ) + + graph_fraction = ( + pynutil.delete("fraction {") + + delete_space + + fraction.inner_graph + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + (hours_unit | non_hours_unit) + ) + + graph = graph_cardinal | graph_decimal | graph_fraction + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..c6aa30ed162405a48b102408fe7350d975bef9d3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/money.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the "License". +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, + insert_space, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money (pt-BR), e.g. + money { currency_maj: "reais" integer_part: "doze" } -> doze reais + money { ... fractional_part: "cinco" currency_min: "centavos" ... } -> doze reais e cinco centavos + + Args: + decimal: DecimalFst verbalizer (for decimal amounts embedded in money) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + scales_data = load_labels(get_abs_path("data/numbers/scales.tsv")) + currency_plural_data = load_labels(get_abs_path("data/money/currency_major_plural.tsv")) + + scale_words = [] + for row in scales_data[1:]: + if len(row) < 2: + continue + one_label = row[0].strip() + plural = row[1].strip() + if not one_label or not plural: + continue + scale_words.extend((one_label.split()[-1], plural)) + + curr_words = [row[1].strip() for row in currency_plural_data if len(row) >= 2 and row[1].strip()] + + scales = pynini.union(*[pynini.accep(w) + NEMO_SPACE for w in scale_words]).optimize() + currencies = pynini.union(*curr_words).optimize() + + maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + min_unit = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + connector_minor = pynutil.insert("e") + insert_space + if not deterministic: + connector_minor |= pynutil.insert("com") + insert_space + + graph_integer = integer_part + NEMO_SPACE + maj + + graph_integer_with_minor = ( + integer_part + + NEMO_SPACE + + maj + + NEMO_SPACE + + connector_minor + + fractional_part + + NEMO_SPACE + + min_unit + + delete_preserve_order + ) + + graph_decimal = decimal.numbers + NEMO_SPACE + maj + + graph_minor = fractional_part + NEMO_SPACE + min_unit + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + graph @= pynini.cdrewrite(pynutil.insert("de") + insert_space, scales, currencies, NEMO_SIGMA) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..5be49ea684a2027e2f04cb599c8587683f62910a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese ordinals, e.g. + ordinal { integer: "primeiro" morphosyntactic_features: "gender_masc" } -> primeiro + ordinal { integer: "primeira" morphosyntactic_features: "gender_fem" } -> primeira (feminine rewrite applied) + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + integer = pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fem_rewrite = pynini.string_file(get_abs_path("data/ordinals/feminine.tsv")) + feminine_rewrite = pynini.cdrewrite( + fem_rewrite, + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]")), + NEMO_SIGMA, + ) + + graph_masc = integer + pynutil.delete(' morphosyntactic_features: "gender_masc"') + graph_fem = (integer @ feminine_rewrite) + pynutil.delete(' morphosyntactic_features: "gender_fem"') + self.fst = self.delete_tokens(pynini.union(graph_masc, graph_fem)).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..294619bf1fcc9f859a9d259d3f64ef00580d42d6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + optional_country_code = pynini.closure( + pynutil.delete('country_code: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete('number_part: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete('"') + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete('extension: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"'), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..c47479cc2082db7ad5e0bc6e86aa207141d353de --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/time.py @@ -0,0 +1,125 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese time, e.g. + time { hours: "catorze" minutes: "trinta" preserve_order: true } -> catorze horas e trinta + time { hours: "um" minutes: "trinta" preserve_order: true } -> uma hora e trinta + time { hours: "dois" minutes: "quinze" preserve_order: true } -> duas horas e quinze + time { hours: "onze" suffix: "da manhã" preserve_order: true } -> onze horas da manhã + time { hours: "vinte e um" minutes: "dezoito" suffix: "da tarde" preserve_order: true } + -> vinte e uma horas e dezoito da tarde + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + quoted = pynini.closure(NEMO_NOT_QUOTE, 1) + + minutes_val = pynutil.delete('minutes: "') + quoted + pynutil.delete('"') + seconds_val = pynutil.delete('seconds: "') + quoted + pynutil.delete('"') + suffix_val = pynutil.delete('suffix: "') + quoted + pynutil.delete('"') + + gap = delete_space + insert_space + suffix_out = pynini.closure(gap + suffix_val, 0, 1) + + hours_default = pynutil.delete('hours: "') + quoted + pynutil.delete('"') + gap + pynutil.insert("horas") + # Match whitespace after the closing quote (same as hours_default's gap) so the path composes + # with minutes/suffix fields; otherwise only the generic "… horas" branch accepts the token. + hours_um = ( + pynutil.delete('hours: "') + + pynutil.delete("um") + + pynutil.delete('"') + + delete_space + + pynutil.insert("uma hora") + ) + hours_dois = ( + pynutil.delete('hours: "') + + pynutil.delete("dois") + + pynutil.delete('"') + + delete_space + + pynutil.insert("duas horas") + ) + hours_vinte_um = ( + pynutil.delete('hours: "') + + pynutil.delete("vinte e um") + + pynutil.delete('"') + + delete_space + + pynutil.insert("vinte e uma horas") + ) + hours_vinte_dois = ( + pynutil.delete('hours: "') + + pynutil.delete("vinte e dois") + + pynutil.delete('"') + + delete_space + + pynutil.insert("vinte e duas horas") + ) + # Prefer feminine hour phrases over the generic ``… horas`` path (tie-break by weight). + hour_phrase = ( + pynutil.add_weight(hours_um, -0.01) + | pynutil.add_weight(hours_dois, -0.01) + | pynutil.add_weight(hours_vinte_um, -0.01) + | pynutil.add_weight(hours_vinte_dois, -0.01) + | hours_default + ).optimize() + + graph_hms = ( + hour_phrase + + insert_space + + pynutil.insert("e") + + insert_space + + minutes_val + + gap + + pynutil.insert("minutos") + + insert_space + + pynutil.insert("e") + + insert_space + + seconds_val + + gap + + pynutil.insert("segundos") + + suffix_out + + delete_preserve_order + ) + + with_minutes = ( + hour_phrase + + insert_space + + pynutil.insert("e") + + insert_space + + minutes_val + + suffix_out + + delete_preserve_order + ) + + hours_only = hour_phrase + suffix_out + delete_preserve_order + + graph = pynini.union(graph_hms, with_minutes, hours_only).optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..2be9a1b279fce2c34835d73764dd76bc4ce50487 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.electronic import ElectronicFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.telephone import TelephoneFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes Portuguese verbalizer grammars (cardinal, ordinal, fraction, decimal). + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(decimal=decimal, deterministic=deterministic) + date = DateFst(deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + graph = ( + fraction.fst + | decimal.fst + | date.fst + | time.fst + | measure.fst + | money.fst + | ordinal.fst + | cardinal.fst + | telephone.fst + | electronic.fst + ) + + self.fst = graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..db2957e47674f4169df9efe6129653033e72a7c5 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.verbalize import VerbalizeFst +from zonos2.vendor.nemo_text_processing.text_normalization.pt.verbalizers.word import WordFst +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire Portuguese sentence, e.g. + tokens { cardinal { integer: "dois" } } tokens { name: "e" } tokens { cardinal { integer: "três" } } -> dois e três + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pt_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..1166dc7dfdcfb4b69188db2bab863bdf5c39861c --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/pt/verbalizers/word.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.pt.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Verbalizes tokens { name: "..." } for the pt-BR pipeline. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/token_parser.py b/zonos2/vendor/nemo_text_processing/text_normalization/token_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..4adcd7d7f048d53288317b97b9cdede908403346 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/token_parser.py @@ -0,0 +1,193 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import string +from collections import OrderedDict +from typing import Dict, List, Union + +PRESERVE_ORDER_KEY = "preserve_order" +EOS = "" + + +class TokenParser: + """ + Parses tokenized/classified text, e.g. 'tokens { money { integer: "20" currency: "$" } } tokens { name: "left"}' + + Args + text: tokenized text + """ + + def __call__(self, text): + """ + Setup function + + Args: + text: text to be parsed + + """ + self.text = text + self.len_text = len(text) + self.char = text[0] # cannot handle empty string + self.index = 0 + + def parse(self) -> List[dict]: + """ + Main function. Implements grammar: + A -> space F space F space F ... space + + Returns list of dictionaries + """ + l = list() + while self.parse_ws(): + token = self.parse_token() + if not token: + break + l.append(token) + return l + + def parse_token(self) -> Dict[str, Union[str, dict]]: + """ + Implements grammar: + F-> no_space KG no_space + + Returns: K, G as dictionary values + """ + d = OrderedDict() + key = self.parse_string_key() + if key is None: + return None + self.parse_ws() + if key == PRESERVE_ORDER_KEY: + self.parse_char(":") + self.parse_ws() + value = self.parse_chars("true") + else: + value = self.parse_token_value() + + d[key] = value + return d + + def parse_token_value(self) -> Union[str, dict]: + """ + Implements grammar: + G-> no_space :"VALUE" no_space | no_space {A} no_space + + Returns: string or dictionary + """ + if self.char == ":": + self.parse_char(":") + self.parse_ws() + self.parse_char("\"") + value_string = self.parse_string_value() + self.parse_char("\"") + return value_string + elif self.char == "{": + d = OrderedDict() + self.parse_char("{") + list_token_dicts = self.parse() + # flatten tokens + for tok_dict in list_token_dicts: + for k, v in tok_dict.items(): + d[k] = v + self.parse_char("}") + return d + else: + raise ValueError() + + def parse_char(self, exp) -> bool: + """ + Parses character + + Args: + exp: character to read in + + Returns true if successful + """ + assert self.char == exp + self.read() + return True + + def parse_chars(self, exp) -> bool: + """ + Parses characters + + Args: + exp: characters to read in + + Returns true if successful + """ + ok = False + for x in exp: + ok |= self.parse_char(x) + return ok + + def parse_string_key(self) -> str: + """ + Parses string key, can only contain ascii and '_' characters + + Returns parsed string key + """ + assert self.char not in string.whitespace and self.char != EOS + incl_criterium = string.ascii_letters + "_" + l = [] + while self.char in incl_criterium: + l.append(self.char) + if not self.read(): + raise ValueError() + + if not l: + return None + return "".join(l) + + def parse_string_value(self) -> str: + """ + Parses string value, ends with quote followed by space + + Returns parsed string value + """ + # assert self.char not in string.whitespace and self.char != EOS + assert self.char != EOS + l = [] + while self.char != "\"" or self.text[self.index + 1] != " ": + l.append(self.char) + if not self.read(): + raise ValueError() + + if not l: + return None + return "".join(l) + + def parse_ws(self): + """ + Deletes whitespaces. + + Returns true if not EOS after parsing + """ + not_eos = self.char != EOS + while not_eos and self.char == " ": + not_eos = self.read() + return not_eos + + def read(self): + """ + Reads in next char. + + Returns true if not EOS + """ + if self.index < self.len_text - 1: # should be unique + self.index += 1 + self.char = self.text[self.index] + return True + self.char = EOS + return False diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/README.md b/zonos2/vendor/nemo_text_processing/text_normalization/zh/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0eedd574d8394a674e0ae62f4d0a2ff6ae2add92 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/README.md @@ -0,0 +1,138 @@ +# Chinese Text Normalization + +## 1. How To Use +``` +python normalize.py --language "zh" --text "text to be normalized" +``` + +## 2. TN Pipeline +There are 3 components in TN pipeline: +* pre-processing (before tagger) +* non-standard word normalization +* post-processing (after verbalizer) + +### 2.1 Pre-Processing +#### Char Width Conversion (全角 -> 半角) +``` +苹果ＣＥＯ宣布发布新ＩＰＨＯＮＥ -> 苹果CEO宣布发布新IPHONE +他说：“我们已经吃过了！”。 -> 他说:"我们已经吃过了!". +``` +* covers English letters, digits, punctuations and some symbols +* the complete mapping table `data/char/fullwidth_to_halfwidth.tsv` + +#### Denylist (Removal) +Sometime you may want to remove certain things like interjections/fillers "啊", "呃" etc +``` +呃这个呃啊我不知道 -> 这个我不知道 +``` +* customizable via `data/denylist/denylist.tsv` + + +### 2.2 Non-Standard-Words(NSW) normalization +#### Numbers +``` +共465篇，约315万字 -> 共四百六十五篇，约三百一十五万字 +共计6.42万人 -> 共计六点四二万人 +同比升高0.6个百分点 -> 同比升高零点六个百分点 +``` + +#### Fraction +``` +总量的1/5以上 -> 总量的五分之一以上 +相当于头发丝的1/16 -> 相当于头发丝的十六分之一 +3/2是一个假分数 -> 二分之三是一个假分数 +``` + +#### Percentage +``` +同比增长6.3% -> 同比增长百分之六点三 +增幅0.4% -> 增幅百分之零点四 +``` + +#### Date +``` +2002/01/28 -> 二零零二年一月二十八日 +2002-01-28 -> 二零零二年一月二十八日 +2002.01.28 -> 二零零二年一月二十八日 +2002/01 -> 二零零二年一月 +``` + +#### Time +``` +8月16号12:00之前 -> 八月十六号十二点之前 +我是5:02开始的 -> 我是五点零二分开始的 +于5:35:36发射 -> 于五点三十五分三十六秒发射 +8:00am准时开会 -> 上午八点准时开会 +``` + +#### Math +``` +比分定格在78:96 -> 比分定格在七十八比九十六 +计算-2的绝对值是2 -> 计算负二的绝对值是二 +±2的平方都是4 -> 正负二的平方都是四 +``` + +#### Money +``` +价格是￥13.5 -> 价格是十三点五元 +价格是$13.5 -> 价格是十三点五美元 +价格是A$13.5 -> 价格是十三点五澳元 +价格是HKD13.5 -> 价格是十三点五港元 +``` + +#### Measure +``` +重达25kg -> 二十五千克 +最高气温38°C -> 最高气温三十八摄氏度 +实际面积120m² -> 实际面积一百二十平方米 +渲染速度10ms一帧 -> 渲染速度十毫秒一帧 +``` + +#### Number series (phone, mobile numbers) +``` +可以打我手机13501234567 -> 可以打我手机一三五零一二三四五六七 +可以拨打12306来咨询 -> 可以拨打一二三零六来咨询 +``` + +#### Erhua(儿化音) Removal +``` +这儿有只鸟儿 -> 这有只鸟 +这事儿好办 -> 这事好办 +我儿子喜欢这地儿 -> 我儿子喜欢这地 +``` +* erhua whitelist is customizable via `data/erhua/whitelist.tsv` + +#### Whitelist(Replacement) +a set of user-defined hard mapping, i.e. exact-string matching & replacement +``` +C E O -> CEO +G P U -> GPU +O2O -> O to O +B2B -> B to B +``` +* customizable via `data/whitelist/default.tsv` + +### 2.3 Post-Processing +#### Punctuation Removal +If enabled, punctuations are removed. + +#### Uppercase or Lowercase Conversion +If enabled, English letters are converted to uppercases / lowercases + +#### Out-Of-Vocabulary(OOV) Tagger +If enabled, OOV chars are tagged with `` and ``, e.g.: +``` +我们안녕 -> 我们안녕 +雪の花 -> 雪の花 +``` +* default charset (national standard) [通用规范汉字表](https://zh.wikipedia.org/wiki/通用规范汉字表) +* you can extend charset via `data/char/charset_extension.tsv` + +## 3. Credits +Author: Zhenxiang MA @ Tsinghua University + +Advisors: [SpeechColab](https://github.com/SpeechColab) organization + +The authors of this work would like to thank: +* The authors of foundational libraries like OpenFst & Pynini +* NeMo team and NeMo open-source community diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_extension.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_extension.tsv new file mode 100644 index 0000000000000000000000000000000000000000..de3ac0a83e7682f70f0a3badef354cf9347919c4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_extension.tsv @@ -0,0 +1,6 @@ +吶 +诶 +屌 +囧 +飚 +屄 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_national_standard_2013_8105.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_national_standard_2013_8105.tsv new file mode 100644 index 0000000000000000000000000000000000000000..06b46c630588cb8f7aba060755e65b612cfb4fde --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/charset_national_standard_2013_8105.tsv @@ -0,0 +1,8105 @@ +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丏 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丢 +两 +严 +丧 +个 +丫 +中 +丰 +串 +临 +丸 +丹 +为 +主 +丽 +举 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乸 +乾 +了 +予 +争 +事 +二 +亍 +于 +亏 +云 +互 +亓 +五 +井 +亘 +亚 +些 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +亶 +亸 +亹 +人 +亿 +什 +仁 +仂 +仃 +仄 +仅 +仆 +仇 +仉 +今 +介 +仍 +从 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +仡 +代 +令 +以 +仨 +仪 +仫 +们 +仰 +仲 +仳 +仵 +件 +价 +任 +份 +仿 +企 +伈 +伉 +伊 +伋 +伍 +伎 +伏 +伐 +休 +众 +优 +伙 +会 +伛 +伞 +伟 +传 +伢 +伣 +伤 +伥 +伦 +伧 +伪 +伫 +伭 +伯 +估 +伲 +伴 +伶 +伸 +伺 +似 +伽 +伾 +佁 +佃 +但 +位 +低 +住 +佐 +佑 +体 +何 +佖 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +佴 +佶 +佸 +佺 +佻 +佼 +佽 +佾 +使 +侁 +侂 +侃 +侄 +侈 +侉 +例 +侍 +侏 +侑 +侔 +侗 +侘 +供 +依 +侠 +侣 +侥 +侦 +侧 +侨 +侩 +侪 +侬 +侮 +侯 +侴 +侵 +侹 +便 +促 +俄 +俅 +俊 +俍 +俎 +俏 +俐 +俑 +俗 +俘 +俙 +俚 +俜 +保 +俞 +俟 +信 +俣 +俦 +俨 +俩 +俪 +俫 +俭 +修 +俯 +俱 +俳 +俵 +俶 +俸 +俺 +俾 +倌 +倍 +倏 +倒 +倓 +倔 +倕 +倘 +候 +倚 +倜 +倞 +借 +倡 +倥 +倦 +倧 +倨 +倩 +倪 +倬 +倭 +倮 +倴 +债 +倻 +值 +倾 +偁 +偃 +假 +偈 +偌 +偎 +偏 +偓 +偕 +做 +停 +偡 +健 +偬 +偭 +偰 +偲 +偶 +偷 +偻 +偾 +偿 +傀 +傃 +傅 +傈 +傉 +傍 +傒 +傕 +傣 +傥 +傧 +储 +傩 +催 +傲 +傺 +傻 +僇 +僎 +像 +僔 +僖 +僚 +僦 +僧 +僬 +僭 +僮 +僰 +僳 +僵 +僻 +儆 +儇 +儋 +儒 +儡 +儦 +儳 +儴 +儿 +兀 +允 +元 +兄 +充 +兆 +先 +光 +克 +免 +兑 +兔 +兕 +兖 +党 +兜 +兢 +入 +全 +八 +公 +六 +兮 +兰 +共 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +冁 +内 +冈 +冉 +册 +再 +冏 +冒 +冔 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冬 +冮 +冯 +冰 +冱 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +净 +凄 +准 +凇 +凉 +凋 +凌 +减 +凑 +凓 +凘 +凛 +凝 +几 +凡 +凤 +凫 +凭 +凯 +凰 +凳 +凶 +凸 +凹 +出 +击 +凼 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +刖 +列 +刘 +则 +刚 +创 +初 +删 +判 +刨 +利 +别 +刬 +刭 +刮 +到 +刳 +制 +刷 +券 +刹 +刺 +刻 +刽 +刿 +剀 +剁 +剂 +剃 +剅 +削 +剋 +剌 +前 +剐 +剑 +剔 +剕 +剖 +剜 +剞 +剟 +剡 +剥 +剧 +剩 +剪 +副 +割 +剽 +剿 +劁 +劂 +劄 +劈 +劐 +劓 +力 +劝 +办 +功 +加 +务 +劢 +劣 +动 +助 +努 +劫 +劬 +劭 +励 +劲 +劳 +劼 +劾 +势 +勃 +勇 +勉 +勋 +勍 +勐 +勒 +勔 +勖 +勘 +勚 +募 +勠 +勤 +勰 +勺 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匏 +匐 +匕 +化 +北 +匙 +匜 +匝 +匠 +匡 +匣 +匦 +匪 +匮 +匹 +区 +医 +匼 +匾 +匿 +十 +千 +卅 +升 +午 +卉 +半 +华 +协 +卑 +卒 +卓 +单 +卖 +南 +博 +卜 +卞 +卟 +占 +卡 +卢 +卣 +卤 +卦 +卧 +卫 +卬 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卺 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厍 +厕 +厖 +厘 +厚 +厝 +原 +厢 +厣 +厥 +厦 +厨 +厩 +厮 +去 +厾 +县 +叁 +参 +叆 +叇 +又 +叉 +及 +友 +双 +反 +发 +叔 +叕 +取 +受 +变 +叙 +叚 +叛 +叟 +叠 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +吣 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吲 +吴 +吵 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呃 +呆 +呇 +呈 +告 +呋 +呐 +呒 +呓 +呔 +呕 +呖 +呗 +员 +呙 +呛 +呜 +呢 +呣 +呤 +呦 +周 +呱 +呲 +味 +呵 +呶 +呷 +呸 +呻 +呼 +命 +咀 +咂 +咄 +咆 +咇 +咉 +咋 +和 +咍 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咙 +咚 +咛 +咝 +咡 +咣 +咤 +咥 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咯 +咱 +咳 +咴 +咸 +咺 +咻 +咽 +咿 +哀 +品 +哂 +哃 +哄 +哆 +哇 +哈 +哉 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哓 +哔 +哕 +哗 +哙 +哚 +哝 +哞 +哟 +哢 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哱 +哲 +哳 +哺 +哼 +哽 +哿 +唁 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唛 +唝 +唠 +唢 +唣 +唤 +唧 +唪 +唬 +售 +唯 +唰 +唱 +唳 +唵 +唷 +唼 +唾 +唿 +啁 +啃 +啄 +商 +啉 +啊 +啐 +啕 +啖 +啜 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啭 +啮 +啰 +啴 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喁 +喂 +喃 +善 +喆 +喇 +喈 +喉 +喊 +喋 +喏 +喑 +喔 +喘 +喙 +喜 +喝 +喟 +喤 +喧 +喱 +喳 +喵 +喷 +喹 +喻 +喽 +喾 +嗄 +嗅 +嗉 +嗌 +嗍 +嗐 +嗑 +嗒 +嗓 +嗔 +嗖 +嗜 +嗝 +嗞 +嗟 +嗡 +嗣 +嗤 +嗥 +嗦 +嗨 +嗪 +嗫 +嗬 +嗯 +嗲 +嗳 +嗵 +嗷 +嗽 +嗾 +嘀 +嘁 +嘈 +嘉 +嘌 +嘎 +嘏 +嘘 +嘚 +嘛 +嘞 +嘟 +嘡 +嘣 +嘤 +嘧 +嘬 +嘭 +嘱 +嘲 +嘴 +嘶 +嘹 +嘻 +嘿 +噀 +噂 +噇 +噌 +噍 +噎 +噔 +噗 +噘 +噙 +噜 +噢 +噤 +器 +噩 +噪 +噫 +噬 +噱 +噶 +噻 +噼 +嚄 +嚅 +嚆 +嚎 +嚏 +嚓 +嚚 +嚣 +嚭 +嚯 +嚷 +嚼 +囊 +囔 +囚 +四 +回 +囟 +因 +囡 +团 +囤 +囫 +园 +困 +囱 +围 +囵 +囷 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +圉 +圊 +圌 +圐 +圙 +圜 +土 +圢 +圣 +在 +圩 +圪 +圫 +圬 +圭 +圮 +圯 +地 +圲 +圳 +圹 +场 +圻 +圾 +址 +坂 +均 +坉 +坊 +坋 +坌 +坍 +坎 +坏 +坐 +坑 +坒 +块 +坚 +坛 +坜 +坝 +坞 +坟 +坠 +坡 +坤 +坥 +坦 +坨 +坩 +坪 +坫 +坬 +坭 +坯 +坰 +坳 +坷 +坻 +坼 +坽 +垂 +垃 +垄 +垆 +垈 +型 +垌 +垍 +垎 +垏 +垒 +垓 +垕 +垙 +垚 +垛 +垞 +垟 +垠 +垡 +垢 +垣 +垤 +垦 +垧 +垩 +垫 +垭 +垮 +垯 +垱 +垲 +垴 +垵 +垸 +垺 +垾 +垿 +埂 +埃 +埆 +埇 +埋 +埌 +城 +埏 +埒 +埔 +埕 +埗 +埘 +埙 +埚 +埝 +域 +埠 +埤 +埪 +埫 +埭 +埯 +埴 +埵 +埸 +培 +基 +埼 +埽 +堂 +堃 +堆 +堇 +堉 +堋 +堌 +堍 +堎 +堐 +堑 +堕 +堙 +堞 +堠 +堡 +堤 +堧 +堨 +堪 +堰 +堲 +堵 +堼 +堽 +堾 +塄 +塅 +塆 +塌 +塍 +塑 +塔 +塘 +塝 +塞 +塥 +填 +塬 +塱 +塾 +墀 +墁 +境 +墅 +墈 +墉 +墐 +墒 +墓 +墕 +墘 +墙 +墚 +增 +墟 +墡 +墣 +墦 +墨 +墩 +墼 +壁 +壅 +壑 +壕 +壤 +士 +壬 +壮 +声 +壳 +壶 +壸 +壹 +处 +备 +复 +夏 +夐 +夔 +夕 +外 +夙 +多 +夜 +够 +夤 +夥 +大 +天 +太 +夫 +夬 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夼 +奁 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +契 +奓 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奡 +奢 +奥 +奭 +女 +奴 +奶 +奸 +她 +好 +妁 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妗 +妘 +妙 +妞 +妣 +妤 +妥 +妧 +妨 +妩 +妪 +妫 +妭 +妮 +妯 +妲 +妹 +妻 +妾 +姆 +姈 +姊 +始 +姐 +姑 +姒 +姓 +委 +姗 +姘 +姚 +姜 +姝 +姞 +姣 +姤 +姥 +姨 +姬 +姮 +姱 +姶 +姹 +姻 +姽 +姿 +娀 +威 +娃 +娄 +娅 +娆 +娇 +娈 +娉 +娌 +娑 +娓 +娘 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娵 +娶 +娼 +婀 +婆 +婉 +婊 +婌 +婍 +婕 +婘 +婚 +婞 +婠 +婢 +婤 +婧 +婪 +婫 +婳 +婴 +婵 +婶 +婷 +婺 +婻 +婼 +婿 +媂 +媄 +媆 +媒 +媓 +媖 +媚 +媛 +媞 +媪 +媭 +媱 +媲 +媳 +媵 +媸 +媾 +嫁 +嫂 +嫄 +嫉 +嫌 +嫒 +嫔 +嫕 +嫖 +嫘 +嫚 +嫜 +嫠 +嫡 +嫣 +嫦 +嫩 +嫪 +嫫 +嫭 +嫱 +嫽 +嬉 +嬖 +嬗 +嬛 +嬥 +嬬 +嬴 +嬷 +嬿 +孀 +孅 +子 +孑 +孓 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +孥 +学 +孩 +孪 +孬 +孰 +孱 +孳 +孵 +孺 +孽 +宁 +它 +宄 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +宠 +审 +客 +宣 +室 +宥 +宦 +宧 +宪 +宫 +宬 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寁 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寝 +寞 +察 +寡 +寤 +寥 +寨 +寮 +寰 +寸 +对 +寺 +寻 +导 +寿 +封 +射 +将 +尉 +尊 +小 +少 +尔 +尕 +尖 +尘 +尚 +尜 +尝 +尢 +尤 +尥 +尧 +尨 +尪 +尬 +就 +尴 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屃 +居 +屈 +屉 +届 +屋 +屎 +屏 +屐 +屑 +展 +屙 +属 +屠 +屡 +屣 +履 +屦 +屯 +山 +屹 +屺 +屼 +屾 +屿 +岁 +岂 +岈 +岊 +岌 +岍 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岜 +岞 +岠 +岢 +岣 +岨 +岩 +岫 +岬 +岭 +岱 +岳 +岵 +岷 +岸 +岽 +岿 +峁 +峂 +峃 +峄 +峋 +峒 +峗 +峘 +峙 +峛 +峡 +峣 +峤 +峥 +峦 +峧 +峨 +峪 +峭 +峰 +峱 +峻 +峿 +崀 +崁 +崂 +崃 +崄 +崆 +崇 +崌 +崎 +崒 +崔 +崖 +崚 +崛 +崞 +崟 +崡 +崤 +崦 +崧 +崩 +崭 +崮 +崴 +崶 +崽 +崾 +崿 +嵁 +嵅 +嵇 +嵊 +嵋 +嵌 +嵎 +嵖 +嵘 +嵚 +嵛 +嵝 +嵩 +嵫 +嵬 +嵯 +嵲 +嵴 +嶂 +嶅 +嶍 +嶒 +嶓 +嶙 +嶝 +嶟 +嶦 +嶲 +嶷 +巅 +巇 +巉 +巍 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +巯 +己 +已 +巳 +巴 +巷 +巽 +巾 +币 +市 +布 +帅 +帆 +师 +希 +帏 +帐 +帑 +帔 +帕 +帖 +帘 +帙 +帚 +帛 +帜 +帝 +帡 +带 +帧 +帨 +席 +帮 +帱 +帷 +常 +帻 +帼 +帽 +幂 +幄 +幅 +幌 +幔 +幕 +幖 +幛 +幞 +幡 +幢 +幪 +干 +平 +年 +并 +幸 +幺 +幻 +幼 +幽 +广 +庄 +庆 +庇 +床 +庋 +序 +庐 +庑 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +庤 +庥 +度 +座 +庭 +庱 +庳 +庵 +庶 +康 +庸 +庹 +庼 +庾 +廆 +廉 +廊 +廋 +廑 +廒 +廓 +廖 +廙 +廛 +廨 +廪 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弆 +弇 +弈 +弊 +弋 +式 +弑 +弓 +引 +弗 +弘 +弛 +弟 +张 +弢 +弥 +弦 +弧 +弨 +弩 +弭 +弯 +弱 +弶 +弸 +弹 +强 +弼 +彀 +归 +当 +录 +彖 +彗 +彘 +彝 +彟 +形 +彤 +彦 +彧 +彩 +彪 +彬 +彭 +彰 +影 +彳 +彷 +役 +彻 +彼 +往 +征 +徂 +径 +待 +徇 +很 +徉 +徊 +律 +徐 +徒 +徕 +得 +徘 +徙 +徛 +徜 +御 +徨 +循 +徭 +微 +徵 +德 +徼 +徽 +心 +必 +忆 +忉 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +忝 +忞 +忠 +忡 +忤 +忧 +忪 +快 +忭 +忮 +忱 +忳 +念 +忸 +忺 +忻 +忽 +忾 +忿 +怀 +态 +怂 +怃 +怄 +怅 +怆 +怊 +怍 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怛 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怩 +怪 +怫 +怯 +怵 +总 +怼 +怿 +恁 +恂 +恃 +恋 +恍 +恐 +恒 +恓 +恔 +恕 +恙 +恚 +恝 +恢 +恣 +恤 +恧 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恶 +恸 +恹 +恺 +恻 +恼 +恽 +恿 +悃 +悄 +悆 +悈 +悉 +悌 +悍 +悒 +悔 +悖 +悚 +悛 +悝 +悟 +悠 +悢 +患 +悦 +您 +悫 +悬 +悭 +悯 +悰 +悱 +悲 +悴 +悸 +悻 +悼 +情 +惆 +惇 +惊 +惋 +惎 +惑 +惔 +惕 +惘 +惙 +惚 +惛 +惜 +惝 +惟 +惠 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +想 +惴 +惶 +惹 +惺 +愀 +愁 +愃 +愆 +愈 +愉 +愍 +愎 +意 +愐 +愔 +愕 +愚 +感 +愠 +愣 +愤 +愦 +愧 +愫 +愭 +愿 +慆 +慈 +慊 +慌 +慎 +慑 +慕 +慝 +慢 +慥 +慧 +慨 +慬 +慭 +慰 +慵 +慷 +憋 +憎 +憔 +憕 +憙 +憧 +憨 +憩 +憬 +憭 +憷 +憺 +憾 +懂 +懈 +懊 +懋 +懑 +懒 +懔 +懦 +懵 +懿 +戆 +戈 +戊 +戋 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +戗 +战 +戚 +戛 +戟 +戡 +戢 +戣 +戤 +戥 +截 +戬 +戭 +戮 +戳 +戴 +户 +戽 +戾 +房 +所 +扁 +扂 +扃 +扅 +扆 +扇 +扈 +扉 +扊 +手 +才 +扎 +扑 +扒 +打 +扔 +托 +扛 +扞 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扳 +扶 +批 +扺 +扼 +扽 +找 +承 +技 +抃 +抄 +抉 +把 +抑 +抒 +抓 +抔 +投 +抖 +抗 +折 +抚 +抛 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +抻 +押 +抽 +抿 +拂 +拃 +拄 +担 +拆 +拇 +拈 +拉 +拊 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +招 +拜 +拟 +拢 +拣 +拤 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拶 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挓 +挖 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挦 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +捂 +捃 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捩 +捭 +据 +捯 +捶 +捷 +捺 +捻 +捽 +掀 +掂 +掇 +授 +掉 +掊 +掌 +掎 +掏 +掐 +排 +掖 +掘 +掞 +掠 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掭 +掮 +掰 +掳 +掴 +掷 +掸 +掺 +掼 +掾 +揄 +揆 +揉 +揍 +描 +提 +插 +揕 +揖 +揠 +握 +揣 +揩 +揪 +揭 +揳 +援 +揶 +揸 +揽 +揿 +搀 +搁 +搂 +搅 +搋 +搌 +搏 +搐 +搒 +搓 +搔 +搛 +搜 +搞 +搠 +搡 +搦 +搪 +搬 +搭 +搴 +携 +搽 +摁 +摄 +摅 +摆 +摇 +摈 +摊 +摏 +摒 +摔 +摘 +摛 +摞 +摧 +摩 +摭 +摴 +摸 +摹 +摽 +撂 +撄 +撅 +撇 +撑 +撒 +撕 +撖 +撙 +撞 +撤 +撩 +撬 +播 +撮 +撰 +撵 +撷 +撸 +撺 +撼 +擀 +擂 +擅 +操 +擎 +擐 +擒 +擘 +擞 +擢 +擤 +擦 +擿 +攀 +攉 +攒 +攘 +攥 +攫 +攮 +支 +收 +攸 +改 +攻 +攽 +放 +政 +故 +效 +敉 +敌 +敏 +救 +敔 +敕 +敖 +教 +敛 +敝 +敞 +敢 +散 +敦 +敩 +敫 +敬 +数 +敲 +整 +敷 +文 +斋 +斌 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斝 +斟 +斠 +斡 +斤 +斥 +斧 +斩 +斫 +断 +斯 +新 +斶 +方 +於 +施 +旁 +旃 +旄 +旅 +旆 +旋 +旌 +旎 +族 +旐 +旒 +旖 +旗 +旞 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旯 +旰 +旱 +旴 +旵 +时 +旷 +旸 +旺 +旻 +旿 +昀 +昂 +昃 +昄 +昆 +昇 +昈 +昉 +昊 +昌 +明 +昏 +昒 +易 +昔 +昕 +昙 +昝 +星 +映 +昡 +昣 +昤 +春 +昧 +昨 +昪 +昫 +昭 +是 +昱 +昳 +昴 +昵 +昶 +昺 +昼 +昽 +显 +晁 +晃 +晅 +晊 +晋 +晌 +晏 +晐 +晒 +晓 +晔 +晕 +晖 +晗 +晙 +晚 +晞 +晟 +晡 +晢 +晤 +晦 +晨 +晪 +晫 +普 +景 +晰 +晱 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暅 +暇 +暌 +暑 +暕 +暖 +暗 +暝 +暧 +暨 +暮 +暲 +暴 +暵 +暶 +暹 +暾 +暿 +曈 +曌 +曙 +曛 +曜 +曝 +曦 +曩 +曰 +曲 +曳 +更 +曷 +曹 +曼 +曾 +替 +最 +月 +有 +朋 +服 +朏 +朐 +朓 +朔 +朕 +朗 +望 +朝 +期 +朦 +木 +未 +末 +本 +札 +术 +朱 +朳 +朴 +朵 +朸 +机 +朽 +杀 +杂 +权 +杄 +杆 +杈 +杉 +杌 +李 +杏 +材 +村 +杓 +杕 +杖 +杙 +杜 +杞 +束 +杠 +条 +来 +杧 +杨 +杩 +杪 +杭 +杯 +杰 +杲 +杳 +杵 +杷 +杻 +杼 +松 +板 +极 +构 +枅 +枇 +枉 +枋 +枍 +析 +枕 +林 +枘 +枚 +果 +枝 +枞 +枢 +枣 +枥 +枧 +枨 +枪 +枫 +枭 +枯 +枰 +枲 +枳 +枵 +架 +枷 +枸 +枹 +柁 +柃 +柄 +柈 +柊 +柏 +某 +柑 +柒 +染 +柔 +柖 +柘 +柙 +柚 +柜 +柝 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柰 +柱 +柳 +柴 +柷 +柽 +柿 +栀 +栅 +标 +栈 +栉 +栊 +栋 +栌 +栎 +栏 +栐 +树 +栒 +栓 +栖 +栗 +栝 +栟 +校 +栩 +株 +栲 +栳 +栴 +样 +核 +根 +栻 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桄 +桅 +框 +案 +桉 +桊 +桌 +桎 +桐 +桑 +桓 +桔 +桕 +桠 +桡 +桢 +档 +桤 +桥 +桦 +桧 +桨 +桩 +桫 +桯 +桲 +桴 +桶 +桷 +桹 +梁 +梃 +梅 +梆 +梌 +梏 +梓 +梗 +梠 +梢 +梣 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梴 +梵 +梼 +梽 +梾 +梿 +检 +棁 +棂 +棉 +棋 +棍 +棐 +棒 +棓 +棕 +棘 +棚 +棠 +棣 +棤 +棨 +棪 +棫 +棬 +森 +棰 +棱 +棵 +棹 +棺 +棻 +棼 +棽 +椀 +椁 +椅 +椆 +椋 +植 +椎 +椐 +椑 +椒 +椓 +椟 +椠 +椤 +椪 +椭 +椰 +椴 +椸 +椹 +椽 +椿 +楂 +楒 +楔 +楗 +楙 +楚 +楝 +楞 +楠 +楣 +楦 +楩 +楪 +楫 +楮 +楯 +楷 +楸 +楹 +楼 +概 +榃 +榄 +榅 +榆 +榇 +榈 +榉 +榍 +榑 +榔 +榕 +榖 +榛 +榜 +榧 +榨 +榫 +榭 +榰 +榱 +榴 +榷 +榻 +槁 +槃 +槊 +槌 +槎 +槐 +槔 +槚 +槛 +槜 +槟 +槠 +槭 +槱 +槲 +槽 +槿 +樊 +樗 +樘 +樟 +模 +樨 +横 +樯 +樱 +樵 +樽 +樾 +橄 +橇 +橐 +橑 +橘 +橙 +橛 +橞 +橡 +橥 +橦 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檑 +檗 +檞 +檠 +檩 +檫 +檬 +櫆 +欂 +欠 +次 +欢 +欣 +欤 +欧 +欲 +欸 +欹 +欺 +欻 +款 +歃 +歅 +歆 +歇 +歉 +歌 +歙 +止 +正 +此 +步 +武 +歧 +歪 +歹 +死 +歼 +殁 +殂 +殃 +殄 +殆 +殇 +殉 +殊 +残 +殍 +殒 +殓 +殖 +殚 +殛 +殡 +殣 +殪 +殳 +殴 +段 +殷 +殿 +毁 +毂 +毅 +毋 +毌 +母 +每 +毐 +毒 +毓 +比 +毕 +毖 +毗 +毙 +毛 +毡 +毪 +毫 +毯 +毳 +毵 +毹 +毽 +氅 +氆 +氇 +氍 +氏 +氐 +民 +氓 +气 +氕 +氖 +氘 +氙 +氚 +氛 +氟 +氡 +氢 +氤 +氦 +氧 +氨 +氩 +氪 +氮 +氯 +氰 +氲 +水 +永 +氾 +氿 +汀 +汁 +求 +汆 +汇 +汈 +汉 +汊 +汋 +汐 +汔 +汕 +汗 +汛 +汜 +汝 +汞 +江 +池 +污 +汤 +汧 +汨 +汩 +汪 +汫 +汭 +汰 +汲 +汴 +汶 +汹 +汽 +汾 +沁 +沂 +沃 +沄 +沅 +沆 +沇 +沈 +沉 +沌 +沏 +沐 +沓 +沔 +沘 +沙 +沚 +沛 +沟 +没 +沣 +沤 +沥 +沦 +沧 +沨 +沩 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +沺 +治 +沼 +沽 +沾 +沿 +泂 +泃 +泄 +泅 +泇 +泉 +泊 +泌 +泐 +泓 +泔 +法 +泖 +泗 +泙 +泚 +泛 +泜 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泺 +泻 +泼 +泽 +泾 +洁 +洄 +洇 +洈 +洋 +洌 +洎 +洑 +洒 +洓 +洗 +洘 +洙 +洚 +洛 +洞 +洢 +洣 +津 +洧 +洨 +洪 +洫 +洭 +洮 +洱 +洲 +洳 +洴 +洵 +洸 +洹 +洺 +活 +洼 +洽 +派 +洿 +流 +浃 +浅 +浆 +浇 +浈 +浉 +浊 +测 +浍 +济 +浏 +浐 +浑 +浒 +浓 +浔 +浕 +浙 +浚 +浛 +浜 +浞 +浟 +浠 +浡 +浣 +浥 +浦 +浩 +浪 +浬 +浭 +浮 +浯 +浰 +浲 +浴 +海 +浸 +浼 +涂 +涄 +涅 +消 +涉 +涌 +涍 +涎 +涐 +涑 +涓 +涔 +涕 +涘 +涛 +涝 +涞 +涟 +涠 +涡 +涢 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涫 +涮 +涯 +液 +涴 +涵 +涸 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淏 +淑 +淖 +淘 +淙 +淜 +淝 +淞 +淟 +淠 +淡 +淤 +淦 +淫 +淬 +淮 +淯 +深 +淳 +淴 +混 +淹 +添 +淼 +清 +渊 +渌 +渍 +渎 +渐 +渑 +渔 +渗 +渚 +渝 +渟 +渠 +渡 +渣 +渤 +渥 +温 +渫 +渭 +港 +渰 +渲 +渴 +游 +渺 +渼 +湃 +湄 +湉 +湍 +湎 +湑 +湓 +湔 +湖 +湘 +湛 +湜 +湝 +湟 +湣 +湫 +湮 +湲 +湴 +湾 +湿 +溁 +溃 +溅 +溆 +溇 +溉 +溍 +溏 +源 +溘 +溚 +溜 +溞 +溟 +溠 +溢 +溥 +溦 +溧 +溪 +溯 +溱 +溲 +溴 +溵 +溶 +溷 +溹 +溺 +溻 +溽 +滁 +滂 +滃 +滆 +滇 +滉 +滋 +滍 +滏 +滑 +滓 +滔 +滕 +滗 +滘 +滚 +滞 +滟 +滠 +满 +滢 +滤 +滥 +滦 +滧 +滨 +滩 +滪 +滫 +滴 +滹 +漂 +漆 +漈 +漉 +漋 +漏 +漓 +演 +漕 +漖 +漠 +漤 +漦 +漩 +漪 +漫 +漭 +漯 +漱 +漳 +漴 +漶 +漷 +漹 +漻 +漼 +漾 +潆 +潇 +潋 +潍 +潏 +潖 +潘 +潜 +潞 +潟 +潢 +潦 +潩 +潭 +潮 +潲 +潴 +潵 +潸 +潺 +潼 +潽 +潾 +澂 +澄 +澈 +澉 +澌 +澍 +澎 +澛 +澜 +澡 +澥 +澧 +澪 +澭 +澳 +澴 +澶 +澹 +澼 +澽 +激 +濂 +濉 +濋 +濑 +濒 +濞 +濠 +濡 +濩 +濮 +濯 +瀌 +瀍 +瀑 +瀔 +瀚 +瀛 +瀣 +瀱 +瀵 +瀹 +瀼 +灈 +灌 +灏 +灞 +火 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +灾 +灿 +炀 +炅 +炆 +炉 +炊 +炌 +炎 +炒 +炔 +炕 +炖 +炘 +炙 +炜 +炝 +炟 +炣 +炫 +炬 +炭 +炮 +炯 +炱 +炳 +炷 +炸 +点 +炻 +炼 +炽 +烀 +烁 +烂 +烃 +烈 +烊 +烔 +烘 +烙 +烛 +烜 +烝 +烟 +烠 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烶 +烷 +烹 +烺 +烻 +烽 +焆 +焉 +焊 +焌 +焐 +焓 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +焞 +焦 +焯 +焰 +焱 +然 +煁 +煃 +煅 +煊 +煋 +煌 +煎 +煓 +煜 +煞 +煟 +煤 +煦 +照 +煨 +煮 +煲 +煳 +煴 +煸 +煺 +煽 +熄 +熇 +熊 +熏 +熔 +熘 +熙 +熛 +熜 +熟 +熠 +熥 +熨 +熬 +熵 +熹 +熻 +燃 +燊 +燋 +燎 +燏 +燔 +燕 +燚 +燠 +燥 +燧 +燮 +燹 +爆 +爇 +爔 +爚 +爝 +爟 +爨 +爪 +爬 +爰 +爱 +爵 +父 +爷 +爸 +爹 +爻 +爽 +爿 +牁 +牂 +片 +版 +牌 +牍 +牒 +牖 +牙 +牚 +牛 +牝 +牟 +牡 +牢 +牤 +牥 +牦 +牧 +物 +牮 +牯 +牲 +牵 +特 +牺 +牻 +牾 +牿 +犀 +犁 +犄 +犇 +犊 +犋 +犍 +犏 +犒 +犟 +犨 +犬 +犯 +犰 +犴 +状 +犷 +犸 +犹 +狁 +狂 +狃 +狄 +狈 +狉 +狍 +狎 +狐 +狒 +狗 +狙 +狝 +狞 +狠 +狡 +狨 +狩 +独 +狭 +狮 +狯 +狰 +狱 +狲 +狳 +狴 +狷 +狸 +狺 +狻 +狼 +猁 +猃 +猄 +猇 +猊 +猎 +猕 +猖 +猗 +猛 +猜 +猝 +猞 +猡 +猢 +猥 +猩 +猪 +猫 +猬 +献 +猯 +猰 +猱 +猴 +猷 +猹 +猺 +猾 +猿 +獍 +獐 +獒 +獗 +獠 +獬 +獭 +獯 +獴 +獾 +玃 +玄 +率 +玉 +王 +玎 +玑 +玒 +玓 +玕 +玖 +玘 +玙 +玚 +玛 +玞 +玟 +玠 +玡 +玢 +玤 +玥 +玦 +玩 +玫 +玭 +玮 +环 +现 +玱 +玲 +玳 +玶 +玷 +玹 +玺 +玻 +玼 +玿 +珀 +珂 +珅 +珇 +珈 +珉 +珊 +珋 +珌 +珍 +珏 +珐 +珑 +珒 +珕 +珖 +珙 +珛 +珝 +珞 +珠 +珢 +珣 +珥 +珦 +珧 +珩 +珪 +珫 +班 +珰 +珲 +珵 +珷 +珸 +珹 +珺 +珽 +琀 +球 +琄 +琅 +理 +琇 +琈 +琉 +琊 +琎 +琏 +琐 +琔 +琚 +琛 +琟 +琡 +琢 +琤 +琥 +琦 +琨 +琪 +琫 +琬 +琭 +琮 +琯 +琰 +琲 +琳 +琴 +琵 +琶 +琼 +瑀 +瑁 +瑂 +瑃 +瑄 +瑅 +瑆 +瑑 +瑓 +瑔 +瑕 +瑖 +瑗 +瑙 +瑚 +瑛 +瑜 +瑝 +瑞 +瑟 +瑢 +瑧 +瑨 +瑬 +瑭 +瑰 +瑱 +瑳 +瑶 +瑷 +瑾 +璀 +璁 +璃 +璆 +璇 +璈 +璋 +璎 +璐 +璒 +璘 +璜 +璞 +璟 +璠 +璥 +璧 +璨 +璩 +璪 +璬 +璮 +璱 +璲 +璺 +瓀 +瓒 +瓖 +瓘 +瓜 +瓞 +瓠 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +瓻 +瓿 +甄 +甍 +甏 +甑 +甓 +甗 +甘 +甚 +甜 +生 +甡 +甥 +甦 +用 +甩 +甪 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +畈 +畋 +界 +畎 +畏 +畔 +畖 +留 +畚 +畛 +畜 +畤 +略 +畦 +番 +畬 +畯 +畲 +畴 +畸 +畹 +畿 +疁 +疃 +疆 +疍 +疏 +疐 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疠 +疡 +疢 +疣 +疤 +疥 +疫 +疬 +疭 +疮 +疯 +疰 +疱 +疲 +疳 +疴 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +痃 +痄 +病 +症 +痈 +痉 +痊 +痍 +痒 +痓 +痔 +痕 +痘 +痛 +痞 +痢 +痣 +痤 +痦 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痼 +痿 +瘀 +瘁 +瘃 +瘅 +瘆 +瘊 +瘌 +瘐 +瘕 +瘗 +瘘 +瘙 +瘛 +瘟 +瘠 +瘢 +瘤 +瘥 +瘦 +瘩 +瘪 +瘫 +瘭 +瘰 +瘳 +瘴 +瘵 +瘸 +瘼 +瘾 +瘿 +癀 +癃 +癌 +癍 +癔 +癖 +癗 +癜 +癞 +癣 +癫 +癯 +癸 +登 +白 +百 +癿 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皕 +皖 +皙 +皛 +皞 +皤 +皦 +皭 +皮 +皱 +皲 +皴 +皿 +盂 +盅 +盆 +盈 +盉 +益 +盍 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盟 +盥 +盦 +目 +盯 +盱 +盲 +直 +盷 +相 +盹 +盼 +盾 +省 +眄 +眇 +眈 +眉 +眊 +看 +眍 +眙 +眚 +真 +眠 +眢 +眦 +眨 +眩 +眬 +眭 +眯 +眵 +眶 +眷 +眸 +眺 +眼 +着 +睁 +睃 +睄 +睇 +睎 +睐 +睑 +睚 +睛 +睡 +睢 +督 +睥 +睦 +睨 +睫 +睬 +睹 +睽 +睾 +睿 +瞀 +瞄 +瞅 +瞋 +瞌 +瞍 +瞎 +瞑 +瞒 +瞟 +瞠 +瞢 +瞥 +瞧 +瞩 +瞪 +瞫 +瞬 +瞭 +瞰 +瞳 +瞵 +瞻 +瞽 +瞿 +矍 +矗 +矛 +矜 +矞 +矢 +矣 +知 +矧 +矩 +矫 +矬 +短 +矮 +矰 +石 +矶 +矸 +矻 +矼 +矾 +矿 +砀 +码 +砂 +砄 +砆 +砉 +砌 +砍 +砑 +砒 +研 +砖 +砗 +砘 +砚 +砜 +砝 +砟 +砠 +砣 +砥 +砧 +砫 +砬 +砭 +砮 +砰 +破 +砵 +砷 +砸 +砹 +砺 +砻 +砼 +砾 +础 +硁 +硅 +硇 +硊 +硌 +硍 +硎 +硐 +硒 +硔 +硕 +硖 +硗 +硙 +硚 +硝 +硪 +硫 +硬 +硭 +确 +硼 +硿 +碃 +碇 +碈 +碉 +碌 +碍 +碎 +碏 +碑 +碓 +碗 +碘 +碚 +碛 +碜 +碟 +碡 +碣 +碥 +碧 +碨 +碰 +碱 +碲 +碳 +碴 +碶 +碹 +碾 +磁 +磅 +磉 +磊 +磋 +磏 +磐 +磔 +磕 +磙 +磜 +磡 +磨 +磬 +磲 +磴 +磷 +磹 +磻 +礁 +礅 +礌 +礓 +礞 +礴 +礵 +示 +礼 +社 +祀 +祁 +祃 +祆 +祇 +祈 +祉 +祊 +祋 +祎 +祏 +祐 +祓 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +祧 +票 +祭 +祯 +祲 +祷 +祸 +祺 +祼 +祾 +禀 +禁 +禄 +禅 +禊 +禋 +福 +禒 +禔 +禘 +禚 +禛 +禤 +禧 +禳 +禹 +禺 +离 +禽 +禾 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秕 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秫 +秬 +秭 +积 +称 +秸 +移 +秽 +秾 +稀 +稂 +稃 +稆 +程 +稌 +稍 +税 +稑 +稔 +稗 +稙 +稚 +稞 +稠 +稣 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穄 +穆 +穑 +穗 +穙 +穜 +穟 +穰 +穴 +究 +穷 +穸 +穹 +空 +穿 +窀 +突 +窃 +窄 +窅 +窈 +窊 +窍 +窎 +窑 +窒 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窣 +窥 +窦 +窨 +窬 +窭 +窳 +窸 +窿 +立 +竑 +竖 +竘 +站 +竞 +竟 +章 +竣 +童 +竦 +竫 +竭 +端 +竹 +竺 +竽 +竿 +笃 +笄 +笆 +笈 +笊 +笋 +笏 +笑 +笔 +笕 +笙 +笛 +笞 +笠 +笤 +笥 +符 +笨 +笪 +笫 +第 +笮 +笯 +笱 +笳 +笸 +笺 +笼 +笾 +筀 +筅 +筇 +等 +筋 +筌 +筏 +筐 +筑 +筒 +答 +策 +筘 +筚 +筛 +筜 +筝 +筠 +筢 +筤 +筥 +筦 +筮 +筱 +筲 +筵 +筶 +筷 +筹 +筻 +筼 +签 +简 +箅 +箍 +箐 +箓 +箔 +箕 +箖 +算 +箜 +管 +箢 +箦 +箧 +箨 +箩 +箪 +箫 +箬 +箭 +箱 +箴 +箸 +篁 +篆 +篇 +篌 +篑 +篓 +篙 +篚 +篝 +篡 +篥 +篦 +篪 +篮 +篯 +篱 +篷 +篼 +篾 +簃 +簇 +簉 +簋 +簌 +簏 +簕 +簖 +簝 +簟 +簠 +簧 +簪 +簰 +簸 +簿 +籀 +籁 +籍 +籥 +米 +籴 +类 +籼 +籽 +粉 +粑 +粒 +粕 +粗 +粘 +粜 +粝 +粞 +粟 +粢 +粤 +粥 +粪 +粮 +粱 +粲 +粳 +粹 +粼 +粽 +精 +粿 +糁 +糅 +糇 +糈 +糊 +糌 +糍 +糒 +糕 +糖 +糗 +糙 +糜 +糟 +糠 +糨 +糯 +糵 +系 +紊 +素 +索 +紧 +紫 +累 +絜 +絮 +絷 +綦 +綮 +縠 +縢 +縻 +繁 +繄 +繇 +纂 +纛 +纠 +纡 +红 +纣 +纤 +纥 +约 +级 +纨 +纩 +纪 +纫 +纬 +纭 +纮 +纯 +纰 +纱 +纲 +纳 +纴 +纵 +纶 +纷 +纸 +纹 +纺 +纻 +纼 +纽 +纾 +线 +绀 +绁 +绂 +练 +组 +绅 +细 +织 +终 +绉 +绊 +绋 +绌 +绍 +绎 +经 +绐 +绑 +绒 +结 +绔 +绕 +绖 +绗 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绠 +绡 +绢 +绣 +绤 +绥 +绦 +继 +绨 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绱 +绲 +绳 +维 +绵 +绶 +绷 +绸 +绹 +绺 +绻 +综 +绽 +绾 +绿 +缀 +缁 +缂 +缃 +缄 +缅 +缆 +缇 +缈 +缉 +缊 +缌 +缎 +缐 +缑 +缒 +缓 +缔 +缕 +编 +缗 +缘 +缙 +缚 +缛 +缜 +缝 +缞 +缟 +缠 +缡 +缢 +缣 +缤 +缥 +缦 +缧 +缨 +缩 +缪 +缫 +缬 +缭 +缮 +缯 +缰 +缱 +缲 +缳 +缴 +缵 +缶 +缸 +缺 +罂 +罄 +罅 +罍 +罐 +网 +罔 +罕 +罗 +罘 +罚 +罟 +罡 +罢 +罨 +罩 +罪 +置 +罱 +署 +罴 +罶 +罹 +罽 +罾 +羁 +羊 +羌 +美 +羑 +羓 +羔 +羕 +羖 +羚 +羝 +羞 +羟 +羡 +群 +羧 +羯 +羰 +羱 +羲 +羸 +羹 +羼 +羽 +羿 +翀 +翁 +翂 +翃 +翅 +翈 +翊 +翌 +翎 +翔 +翕 +翘 +翙 +翚 +翛 +翟 +翠 +翡 +翥 +翦 +翩 +翮 +翯 +翰 +翱 +翳 +翷 +翻 +翼 +翾 +耀 +老 +考 +耄 +者 +耆 +耇 +耋 +而 +耍 +耏 +耐 +耑 +耒 +耔 +耕 +耖 +耗 +耘 +耙 +耜 +耠 +耢 +耤 +耥 +耦 +耧 +耨 +耩 +耪 +耰 +耱 +耳 +耵 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聃 +聆 +聊 +聋 +职 +聍 +聒 +联 +聘 +聚 +聩 +聪 +聱 +聿 +肃 +肄 +肆 +肇 +肉 +肋 +肌 +肓 +肖 +肘 +肚 +肛 +肝 +肟 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肫 +肭 +肮 +肯 +肱 +育 +肴 +肷 +肸 +肺 +肼 +肽 +肾 +肿 +胀 +胁 +胂 +胃 +胄 +胆 +胈 +背 +胍 +胎 +胖 +胗 +胙 +胚 +胛 +胜 +胝 +胞 +胠 +胡 +胣 +胤 +胥 +胧 +胨 +胩 +胪 +胫 +胬 +胭 +胯 +胰 +胱 +胲 +胳 +胴 +胶 +胸 +胺 +胼 +能 +脂 +脆 +脉 +脊 +脍 +脎 +脏 +脐 +脑 +脒 +脓 +脔 +脖 +脘 +脚 +脞 +脟 +脩 +脬 +脯 +脱 +脲 +脶 +脸 +脾 +脿 +腆 +腈 +腊 +腋 +腌 +腐 +腑 +腒 +腓 +腔 +腕 +腘 +腙 +腚 +腠 +腥 +腧 +腨 +腩 +腭 +腮 +腯 +腰 +腱 +腴 +腹 +腺 +腻 +腼 +腽 +腾 +腿 +膀 +膂 +膈 +膊 +膏 +膑 +膘 +膙 +膛 +膜 +膝 +膦 +膨 +膳 +膺 +膻 +臀 +臂 +臃 +臆 +臊 +臌 +臑 +臜 +臣 +臧 +自 +臬 +臭 +至 +致 +臻 +臼 +臾 +舀 +舁 +舂 +舄 +舅 +舆 +舌 +舍 +舐 +舒 +舔 +舛 +舜 +舞 +舟 +舠 +舢 +舣 +舥 +航 +舫 +般 +舭 +舯 +舰 +舱 +舲 +舳 +舴 +舵 +舶 +舷 +舸 +船 +舻 +舾 +艄 +艅 +艇 +艉 +艋 +艎 +艏 +艘 +艚 +艟 +艨 +艮 +良 +艰 +色 +艳 +艴 +艺 +艽 +艾 +艿 +节 +芃 +芄 +芈 +芊 +芋 +芍 +芎 +芏 +芑 +芒 +芗 +芘 +芙 +芜 +芝 +芟 +芠 +芡 +芣 +芤 +芥 +芦 +芨 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +芰 +花 +芳 +芴 +芷 +芸 +芹 +芼 +芽 +芾 +苁 +苄 +苇 +苈 +苉 +苊 +苋 +苌 +苍 +苎 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苘 +苛 +苜 +苞 +苟 +苠 +苡 +苣 +苤 +若 +苦 +苧 +苫 +苯 +英 +苴 +苷 +苹 +苻 +苾 +茀 +茁 +茂 +范 +茄 +茅 +茆 +茈 +茉 +茋 +茌 +茎 +茏 +茑 +茓 +茔 +茕 +茗 +茚 +茛 +茜 +茝 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茳 +茴 +茵 +茶 +茸 +茹 +茺 +茼 +茽 +荀 +荁 +荃 +荄 +荆 +荇 +草 +荏 +荐 +荑 +荒 +荓 +荔 +荖 +荙 +荚 +荛 +荜 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荦 +荧 +荨 +荩 +荪 +荫 +荬 +荭 +荮 +药 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莎 +莒 +莓 +莘 +莙 +莛 +莜 +莝 +莞 +莠 +莨 +莩 +莪 +莫 +莰 +莱 +莲 +莳 +莴 +莶 +获 +莸 +莹 +莺 +莼 +莽 +莿 +菀 +菁 +菂 +菅 +菇 +菉 +菊 +菌 +菍 +菏 +菔 +菖 +菘 +菜 +菝 +菟 +菠 +菡 +菥 +菩 +菪 +菰 +菱 +菲 +菹 +菼 +菽 +萁 +萃 +萄 +萆 +萋 +萌 +萍 +萎 +萏 +萑 +萘 +萚 +萜 +萝 +萣 +萤 +营 +萦 +萧 +萨 +萩 +萱 +萳 +萸 +萹 +萼 +落 +葆 +葎 +葑 +葖 +著 +葙 +葚 +葛 +葜 +葡 +董 +葩 +葫 +葬 +葭 +葰 +葱 +葳 +葴 +葵 +葶 +葸 +葺 +蒂 +蒄 +蒇 +蒈 +蒉 +蒋 +蒌 +蒎 +蒐 +蒗 +蒙 +蒜 +蒟 +蒡 +蒨 +蒯 +蒱 +蒲 +蒴 +蒸 +蒹 +蒺 +蒻 +蒽 +蒿 +蓁 +蓂 +蓄 +蓇 +蓉 +蓊 +蓍 +蓏 +蓐 +蓑 +蓓 +蓖 +蓝 +蓟 +蓠 +蓢 +蓣 +蓥 +蓦 +蓬 +蓰 +蓼 +蓿 +蔀 +蔃 +蔈 +蔊 +蔌 +蔑 +蔓 +蔗 +蔚 +蔟 +蔡 +蔫 +蔬 +蔷 +蔸 +蔹 +蔺 +蔻 +蔼 +蔽 +蕃 +蕈 +蕉 +蕊 +蕖 +蕗 +蕙 +蕞 +蕤 +蕨 +蕰 +蕲 +蕴 +蕹 +蕺 +蕻 +蕾 +薁 +薄 +薅 +薇 +薏 +薛 +薜 +薢 +薤 +薨 +薪 +薮 +薯 +薰 +薳 +薷 +薸 +薹 +薿 +藁 +藉 +藏 +藐 +藓 +藕 +藜 +藟 +藠 +藤 +藦 +藨 +藩 +藻 +藿 +蘅 +蘑 +蘖 +蘘 +蘧 +蘩 +蘸 +蘼 +虎 +虏 +虐 +虑 +虒 +虓 +虔 +虚 +虞 +虢 +虤 +虫 +虬 +虮 +虱 +虷 +虸 +虹 +虺 +虻 +虼 +虽 +虾 +虿 +蚀 +蚁 +蚂 +蚄 +蚆 +蚊 +蚋 +蚌 +蚍 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚧 +蚨 +蚩 +蚪 +蚬 +蚯 +蚰 +蚱 +蚲 +蚴 +蚶 +蚺 +蛀 +蛃 +蛄 +蛆 +蛇 +蛉 +蛊 +蛋 +蛎 +蛏 +蛐 +蛑 +蛔 +蛘 +蛙 +蛛 +蛞 +蛟 +蛤 +蛩 +蛭 +蛮 +蛰 +蛱 +蛲 +蛳 +蛴 +蛸 +蛹 +蛾 +蜀 +蜂 +蜃 +蜇 +蜈 +蜉 +蜊 +蜍 +蜎 +蜐 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜞 +蜡 +蜢 +蜣 +蜥 +蜩 +蜮 +蜱 +蜴 +蜷 +蜻 +蜾 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝓 +蝗 +蝘 +蝙 +蝠 +蝣 +蝤 +蝥 +蝮 +蝰 +蝲 +蝴 +蝶 +蝻 +蝼 +蝽 +蝾 +螂 +螃 +螅 +螈 +螋 +融 +螗 +螟 +螠 +螣 +螨 +螫 +螬 +螭 +螯 +螱 +螳 +螵 +螺 +螽 +蟀 +蟆 +蟊 +蟋 +蟏 +蟑 +蟒 +蟛 +蟠 +蟥 +蟪 +蟫 +蟮 +蟹 +蟾 +蠃 +蠊 +蠋 +蠓 +蠕 +蠖 +蠡 +蠢 +蠲 +蠹 +蠼 +血 +衃 +衄 +衅 +行 +衍 +衎 +衒 +衔 +街 +衙 +衠 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衽 +衾 +衿 +袁 +袂 +袄 +袅 +袆 +袈 +袋 +袍 +袒 +袖 +袗 +袜 +袢 +袤 +袪 +被 +袭 +袯 +袱 +袷 +袼 +裁 +裂 +装 +裆 +裈 +裉 +裎 +裒 +裔 +裕 +裘 +裙 +裛 +裟 +裢 +裣 +裤 +裥 +裨 +裰 +裱 +裳 +裴 +裸 +裹 +裼 +裾 +褂 +褊 +褐 +褒 +褓 +褕 +褙 +褚 +褛 +褟 +褡 +褥 +褪 +褫 +褯 +褰 +褴 +褶 +襁 +襄 +襕 +襚 +襜 +襞 +襟 +襦 +襫 +襻 +西 +要 +覃 +覆 +见 +观 +觃 +规 +觅 +视 +觇 +览 +觉 +觊 +觋 +觌 +觎 +觏 +觐 +觑 +角 +觖 +觚 +觜 +觞 +觟 +解 +觥 +触 +觫 +觭 +觯 +觱 +觳 +觿 +言 +訄 +訇 +訚 +訾 +詈 +詟 +詹 +誉 +誊 +誓 +謇 +警 +譬 +计 +订 +讣 +认 +讥 +讦 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讱 +讲 +讳 +讴 +讵 +讶 +讷 +许 +讹 +论 +讻 +讼 +讽 +设 +访 +诀 +证 +诂 +诃 +评 +诅 +识 +诇 +诈 +诉 +诊 +诋 +诌 +词 +诎 +诏 +诐 +译 +诒 +诓 +诔 +试 +诖 +诗 +诘 +诙 +诚 +诛 +诜 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诨 +诩 +诫 +诬 +语 +诮 +误 +诰 +诱 +诲 +诳 +说 +诵 +请 +诸 +诹 +诺 +读 +诼 +诽 +课 +诿 +谀 +谁 +谂 +调 +谄 +谅 +谆 +谇 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谖 +谗 +谙 +谚 +谛 +谜 +谝 +谞 +谟 +谠 +谡 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谫 +谬 +谭 +谮 +谯 +谰 +谱 +谲 +谳 +谴 +谵 +谶 +谷 +谼 +谿 +豁 +豆 +豇 +豉 +豌 +豕 +豚 +象 +豢 +豨 +豪 +豫 +豮 +豳 +豸 +豹 +豺 +貂 +貅 +貆 +貉 +貊 +貌 +貔 +貘 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贳 +贴 +贵 +贶 +贷 +贸 +费 +贺 +贻 +贼 +贽 +贾 +贿 +赀 +赁 +赂 +赃 +资 +赅 +赆 +赇 +赈 +赉 +赊 +赋 +赌 +赍 +赎 +赏 +赐 +赑 +赒 +赓 +赔 +赕 +赖 +赗 +赘 +赙 +赚 +赛 +赜 +赝 +赞 +赟 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赪 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +趄 +超 +越 +趋 +趑 +趔 +趟 +趣 +趯 +趱 +足 +趴 +趵 +趸 +趺 +趼 +趾 +趿 +跂 +跃 +跄 +跆 +跋 +跌 +跎 +跏 +跐 +跑 +跖 +跗 +跚 +跛 +距 +跞 +跟 +跣 +跤 +跨 +跪 +跬 +路 +跱 +跳 +践 +跶 +跷 +跸 +跹 +跺 +跻 +跽 +踅 +踉 +踊 +踌 +踏 +踒 +踔 +踝 +踞 +踟 +踢 +踣 +踦 +踩 +踪 +踬 +踮 +踯 +踱 +踵 +踶 +踹 +踺 +踽 +蹀 +蹁 +蹂 +蹄 +蹅 +蹇 +蹈 +蹉 +蹊 +蹋 +蹐 +蹑 +蹒 +蹙 +蹚 +蹜 +蹢 +蹦 +蹩 +蹬 +蹭 +蹯 +蹰 +蹲 +蹴 +蹶 +蹼 +蹽 +蹾 +蹿 +躁 +躅 +躇 +躏 +躐 +躔 +躜 +躞 +身 +躬 +躯 +躲 +躺 +车 +轧 +轨 +轩 +轪 +轫 +转 +轭 +轮 +软 +轰 +轱 +轲 +轳 +轴 +轵 +轶 +轷 +轸 +轹 +轺 +轻 +轼 +载 +轾 +轿 +辀 +辁 +辂 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辋 +辌 +辍 +辎 +辏 +辐 +辑 +辒 +输 +辔 +辕 +辖 +辗 +辘 +辙 +辚 +辛 +辜 +辞 +辟 +辣 +辨 +辩 +辫 +辰 +辱 +边 +辽 +达 +辿 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +迓 +返 +迕 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迨 +迩 +迪 +迫 +迭 +迮 +述 +迳 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逄 +逅 +逆 +选 +逊 +逋 +逍 +透 +逐 +逑 +递 +途 +逖 +逗 +通 +逛 +逝 +逞 +速 +造 +逡 +逢 +逦 +逭 +逮 +逯 +逴 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遄 +遆 +遇 +遍 +遏 +遐 +遑 +遒 +道 +遗 +遘 +遛 +遢 +遣 +遥 +遨 +遭 +遮 +遴 +遵 +遹 +遽 +避 +邀 +邂 +邃 +邈 +邋 +邑 +邓 +邕 +邗 +邘 +邙 +邛 +邝 +邠 +邡 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邲 +邳 +邴 +邵 +邶 +邸 +邹 +邺 +邻 +邽 +邾 +邿 +郁 +郃 +郄 +郅 +郇 +郈 +郊 +郎 +郏 +郐 +郑 +郓 +郗 +郚 +郛 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郪 +郫 +郭 +郯 +郴 +郸 +都 +郾 +郿 +鄀 +鄂 +鄃 +鄄 +鄅 +鄌 +鄑 +鄗 +鄘 +鄙 +鄚 +鄜 +鄞 +鄠 +鄢 +鄣 +鄫 +鄯 +鄱 +鄹 +酂 +酃 +酅 +酆 +酉 +酊 +酋 +酌 +配 +酎 +酏 +酐 +酒 +酗 +酚 +酝 +酞 +酡 +酢 +酣 +酤 +酥 +酦 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酲 +酴 +酵 +酶 +酷 +酸 +酹 +酺 +酽 +酾 +酿 +醅 +醇 +醉 +醋 +醌 +醍 +醐 +醑 +醒 +醚 +醛 +醢 +醨 +醪 +醭 +醮 +醯 +醴 +醵 +醺 +醾 +采 +釉 +释 +里 +重 +野 +量 +釐 +金 +釜 +鉴 +銎 +銮 +鋆 +鋈 +錾 +鍪 +鎏 +鏊 +鏖 +鐾 +鑫 +钆 +钇 +针 +钉 +钊 +钋 +钌 +钍 +钎 +钏 +钐 +钒 +钓 +钔 +钕 +钖 +钗 +钘 +钙 +钚 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钪 +钫 +钬 +钭 +钮 +钯 +钰 +钱 +钲 +钳 +钴 +钵 +钷 +钹 +钺 +钻 +钼 +钽 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铈 +铉 +铊 +铋 +铌 +铍 +铎 +铏 +铐 +铑 +铒 +铕 +铖 +铗 +铘 +铙 +铚 +铛 +铜 +铝 +铞 +铟 +铠 +铡 +铢 +铣 +铤 +铥 +铧 +铨 +铩 +铪 +铫 +铬 +铭 +铮 +铯 +铰 +铱 +铲 +铳 +铴 +铵 +银 +铷 +铸 +铹 +铺 +铻 +铼 +铽 +链 +铿 +销 +锁 +锂 +锃 +锄 +锅 +锆 +锇 +锈 +锉 +锊 +锋 +锌 +锍 +锎 +锏 +锐 +锑 +锒 +锓 +锔 +锕 +锖 +锗 +锘 +错 +锚 +锛 +锜 +锝 +锞 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锧 +锨 +锩 +锪 +锫 +锬 +锭 +键 +锯 +锰 +锱 +锲 +锳 +锴 +锵 +锶 +锷 +锸 +锹 +锺 +锻 +锼 +锽 +锾 +锿 +镀 +镁 +镂 +镃 +镄 +镅 +镆 +镇 +镈 +镉 +镊 +镋 +镌 +镍 +镎 +镏 +镐 +镑 +镒 +镓 +镔 +镕 +镖 +镗 +镘 +镚 +镛 +镜 +镝 +镞 +镠 +镡 +镢 +镣 +镤 +镥 +镦 +镧 +镨 +镩 +镪 +镫 +镬 +镭 +镮 +镯 +镰 +镱 +镲 +镳 +镴 +镵 +镶 +长 +门 +闩 +闪 +闫 +闭 +问 +闯 +闰 +闱 +闲 +闳 +间 +闵 +闶 +闷 +闸 +闹 +闺 +闻 +闼 +闽 +闾 +闿 +阀 +阁 +阂 +阃 +阄 +阅 +阆 +阇 +阈 +阉 +阊 +阋 +阌 +阍 +阎 +阏 +阐 +阑 +阒 +阔 +阕 +阖 +阗 +阘 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阼 +阽 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陉 +陋 +陌 +降 +陎 +限 +陑 +陔 +陕 +陛 +陞 +陟 +陡 +院 +除 +陧 +陨 +险 +陪 +陬 +陲 +陴 +陵 +陶 +陷 +隃 +隅 +隆 +隈 +隋 +隍 +随 +隐 +隔 +隗 +隘 +隙 +障 +隧 +隩 +隰 +隳 +隶 +隹 +隺 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雊 +雌 +雍 +雎 +雏 +雒 +雕 +雠 +雨 +雩 +雪 +雯 +雱 +雳 +零 +雷 +雹 +雾 +需 +霁 +霄 +霅 +霆 +震 +霈 +霉 +霍 +霎 +霏 +霓 +霖 +霜 +霞 +霨 +霪 +霭 +霰 +露 +霸 +霹 +霾 +青 +靓 +靖 +静 +靛 +非 +靠 +靡 +面 +靥 +革 +靬 +靰 +靳 +靴 +靶 +靸 +靺 +靼 +靽 +靿 +鞁 +鞅 +鞋 +鞍 +鞑 +鞒 +鞔 +鞘 +鞠 +鞡 +鞣 +鞧 +鞨 +鞫 +鞬 +鞭 +鞮 +鞯 +鞲 +鞳 +鞴 +韂 +韦 +韧 +韨 +韩 +韪 +韫 +韬 +韭 +音 +韵 +韶 +页 +顶 +顷 +顸 +项 +顺 +须 +顼 +顽 +顾 +顿 +颀 +颁 +颂 +颃 +预 +颅 +领 +颇 +颈 +颉 +颊 +颋 +颌 +颍 +颎 +颏 +颐 +频 +颓 +颔 +颖 +颗 +题 +颙 +颚 +颛 +颜 +额 +颞 +颟 +颠 +颡 +颢 +颤 +颥 +颦 +颧 +风 +飏 +飐 +飑 +飒 +飓 +飔 +飕 +飗 +飘 +飙 +飞 +食 +飧 +飨 +餍 +餐 +餮 +饔 +饕 +饥 +饧 +饨 +饩 +饪 +饫 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饳 +饴 +饵 +饶 +饷 +饸 +饹 +饺 +饻 +饼 +饽 +饿 +馁 +馃 +馄 +馅 +馆 +馇 +馈 +馉 +馊 +馋 +馌 +馍 +馏 +馐 +馑 +馒 +馓 +馔 +馕 +首 +馗 +馘 +香 +馝 +馞 +馥 +馧 +馨 +马 +驭 +驮 +驯 +驰 +驱 +驲 +驳 +驴 +驵 +驶 +驷 +驸 +驹 +驺 +驻 +驼 +驽 +驾 +驿 +骀 +骁 +骂 +骃 +骄 +骅 +骆 +骇 +骈 +骉 +骊 +骋 +验 +骍 +骎 +骏 +骐 +骑 +骒 +骓 +骕 +骖 +骗 +骘 +骙 +骚 +骛 +骜 +骝 +骞 +骟 +骠 +骡 +骢 +骣 +骤 +骥 +骦 +骧 +骨 +骰 +骱 +骶 +骷 +骸 +骺 +骼 +髀 +髁 +髂 +髃 +髅 +髋 +髌 +髎 +髑 +髓 +高 +髡 +髢 +髦 +髫 +髭 +髯 +髹 +髻 +髽 +鬃 +鬈 +鬏 +鬒 +鬓 +鬘 +鬟 +鬣 +鬯 +鬲 +鬶 +鬷 +鬻 +鬼 +魁 +魂 +魃 +魄 +魅 +魆 +魇 +魈 +魉 +魋 +魍 +魏 +魑 +魔 +鱼 +鱽 +鱾 +鱿 +鲀 +鲁 +鲂 +鲃 +鲅 +鲆 +鲇 +鲈 +鲉 +鲊 +鲋 +鲌 +鲍 +鲎 +鲏 +鲐 +鲑 +鲒 +鲔 +鲕 +鲖 +鲗 +鲘 +鲙 +鲚 +鲛 +鲜 +鲝 +鲞 +鲟 +鲠 +鲡 +鲢 +鲣 +鲤 +鲥 +鲦 +鲧 +鲨 +鲩 +鲪 +鲫 +鲬 +鲭 +鲮 +鲯 +鲰 +鲱 +鲲 +鲳 +鲴 +鲵 +鲷 +鲸 +鲹 +鲺 +鲻 +鲼 +鲽 +鲾 +鲿 +鳀 +鳁 +鳂 +鳃 +鳄 +鳅 +鳇 +鳈 +鳉 +鳊 +鳌 +鳍 +鳎 +鳏 +鳐 +鳑 +鳒 +鳓 +鳔 +鳕 +鳖 +鳗 +鳘 +鳙 +鳚 +鳛 +鳜 +鳝 +鳞 +鳟 +鳠 +鳡 +鳢 +鳣 +鳤 +鸟 +鸠 +鸡 +鸢 +鸣 +鸤 +鸥 +鸦 +鸧 +鸨 +鸩 +鸪 +鸫 +鸬 +鸭 +鸮 +鸯 +鸰 +鸱 +鸲 +鸳 +鸵 +鸶 +鸷 +鸸 +鸹 +鸺 +鸻 +鸼 +鸽 +鸾 +鸿 +鹀 +鹁 +鹂 +鹃 +鹄 +鹅 +鹆 +鹇 +鹈 +鹉 +鹊 +鹋 +鹌 +鹍 +鹎 +鹏 +鹐 +鹑 +鹒 +鹔 +鹕 +鹖 +鹗 +鹘 +鹙 +鹚 +鹛 +鹜 +鹝 +鹞 +鹟 +鹠 +鹡 +鹢 +鹣 +鹤 +鹦 +鹧 +鹨 +鹩 +鹪 +鹫 +鹬 +鹭 +鹮 +鹯 +鹰 +鹱 +鹲 +鹳 +鹴 +鹾 +鹿 +麀 +麂 +麇 +麈 +麋 +麑 +麒 +麓 +麖 +麝 +麟 +麦 +麸 +麹 +麻 +麽 +麾 +黄 +黇 +黉 +黍 +黎 +黏 +黑 +黔 +默 +黛 +黜 +黝 +黟 +黠 +黡 +黢 +黥 +黧 +黩 +黪 +黯 +黹 +黻 +黼 +黾 +鼋 +鼍 +鼎 +鼐 +鼒 +鼓 +鼗 +鼙 +鼠 +鼢 +鼩 +鼫 +鼬 +鼯 +鼱 +鼷 +鼹 +鼻 +鼽 +鼾 +齁 +齇 +齉 +齐 +齑 +齿 +龀 +龁 +龂 +龃 +龄 +龅 +龆 +龇 +龈 +龉 +龊 +龋 +龌 +龙 +龚 +龛 +龟 +龠 +龢 +鿍 +鿎 +鿏 +㑇 +㑊 +㕮 +㘎 +㙍 +㙘 +㙦 +㛃 +㛚 +㛹 +㟃 +㠇 +㠓 +㤘 +㥄 +㧐 +㧑 +㧟 +㫰 +㬊 +㬎 +㬚 +㭎 +㭕 +㮾 +㰀 +㳇 +㳘 +㳚 +㴔 +㵐 +㶲 +㸆 +㸌 +㺄 +㻬 +㽏 +㿠 +䁖 +䂮 +䃅 +䃎 +䅟 +䌹 +䎃 +䎖 +䏝 +䏡 +䏲 +䐃 +䓖 +䓛 +䓨 +䓫 +䓬 +䗖 +䗛 +䗪 +䗴 +䜣 +䝙 +䢺 +䢼 +䣘 +䥽 +䦃 +䲟 +䲠 +䲢 +䴓 +䴔 +䴕 +䴖 +䴗 +䴘 +䴙 +䶮 +𠅤 +𠙶 +𠳐 +𡎚 +𡐓 +𣗋 +𣲗 +𣲘 +𣸣 +𤧛 +𤩽 +𤫉 +𥔲 +𥕢 +𥖨 +𥻗 +𦈡 +𦒍 +𦙶 +𦝼 +𦭜 +𦰡 +𧿹 +𨐈 +𨙸 +𨚕 +𨟠 +𨭉 +𨱇 +𨱏 +𨱑 +𨱔 +𨺙 +𩽾 +𩾃 +𩾌 +𪟝 +𪣻 +𪤗 +𪨰 +𪨶 +𪩘 +𪾢 +𫄧 +𫄨 +𫄷 +𫄸 +𫇭 +𫌀 +𫍣 +𫍯 +𫍲 +𫍽 +𫐄 +𫐐 +𫐓 +𫑡 +𫓧 +𫓯 +𫓶 +𫓹 +𫔍 +𫔎 +𫔶 +𫖮 +𫖯 +𫖳 +𫗧 +𫗴 +𫘜 +𫘝 +𫘦 +𫘧 +𫘨 +𫘪 +𫘬 +𫚕 +𫚖 +𫚭 +𫛭 +𫞩 +𫟅 +𫟦 +𫟹 +𫟼 +𫠆 +𫠊 +𫠜 +𫢸 +𫫇 +𫭟 +𫭢 +𫭼 +𫮃 +𫰛 +𫵷 +𫶇 +𫷷 +𫸩 +𬀩 +𬀪 +𬂩 +𬃊 +𬇕 +𬇙 +𬇹 +𬉼 +𬊈 +𬊤 +𬌗 +𬍛 +𬍡 +𬍤 +𬒈 +𬒔 +𬒗 +𬕂 +𬘓 +𬘘 +𬘡 +𬘩 +𬘫 +𬘬 +𬘭 +𬘯 +𬙂 +𬙊 +𬙋 +𬜬 +𬜯 +𬞟 +𬟁 +𬟽 +𬣙 +𬣞 +𬣡 +𬣳 +𬤇 +𬤊 +𬤝 +𬨂 +𬨎 +𬩽 +𬪩 +𬬩 +𬬭 +𬬮 +𬬱 +𬬸 +𬬹 +𬬻 +𬬿 +𬭁 +𬭊 +𬭎 +𬭚 +𬭛 +𬭤 +𬭩 +𬭬 +𬭯 +𬭳 +𬭶 +𬭸 +𬭼 +𬮱 +𬮿 +𬯀 +𬯎 +𬱖 +𬱟 +𬳵 +𬳶 +𬳽 +𬳿 +𬴂 +𬴃 +𬴊 +𬶋 +𬶍 +𬶏 +𬶐 +𬶟 +𬶠 +𬶨 +𬶭 +𬶮 +𬷕 +𬸘 +𬸚 +𬸣 +𬸦 +𬸪 +𬹼 +𬺈 +𬺓 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/fullwidth_to_halfwidth.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/fullwidth_to_halfwidth.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7b3fd042c8d626a1e8699bd6f28f90fc60831012 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/fullwidth_to_halfwidth.tsv @@ -0,0 +1,92 @@ +！ ! +＂ " +＃ # +＄ $ +％ % +＆ & +＇ ' +（ ( +） ) +＊ * +＋ + +， , +－ - +． . +／ / +０ 0 +１ 1 +２ 2 +３ 3 +４ 4 +５ 5 +６ 6 +７ 7 +８ 8 +９ 9 +； ; +＜ < +＝ = +＞ > +？ ? +＠ @ +Ａ A +Ｂ B +Ｃ C +Ｄ D +Ｅ E +Ｆ F +Ｇ G +Ｈ H +Ｉ I +Ｊ J +Ｋ K +Ｌ L +Ｍ M +Ｎ N +Ｏ O +Ｐ P +Ｑ Q +Ｒ R +Ｓ S +Ｔ T +Ｕ U +Ｖ V +Ｗ W +Ｘ X +Ｙ Y +Ｚ Z +＼ \ +＾ ^ +＿ _ +｀ ` +ａ a +ｂ b +ｃ c +ｄ d +ｅ e +ｆ f +ｇ g +ｈ h +ｉ i +ｊ j +ｋ k +ｌ l +ｍ m +ｎ n +ｏ o +ｐ p +ｑ q +ｒ r +ｓ s +ｔ t +ｕ u +ｖ v +ｗ w +ｘ x +ｙ y +ｚ z +｛ { +｜ | +： : +｝ } +～ ~ diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/oov_tags.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/oov_tags.tsv new file mode 100644 index 0000000000000000000000000000000000000000..cd6c56f5370054ed4a59a4dba43bcb34cd773faa --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/oov_tags.tsv @@ -0,0 +1 @@ + diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3848d54f9fba3ed4c2dc32f94cdff31a58912994 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv @@ -0,0 +1,74 @@ +！ +？ +｡ +。 +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +／ +： +； +＜ +＝ +＞ +＠ +［ +＼ +］ +＾ +＿ +｀ +｛ +｜ +｝ +～ +｟ +｠ +｢ +｣ +､ +、 +〃 +》 +「 +」 +『 +』 +【 +】 +〔 +〕 +〖 +〗 +〘 +〙 +〚 +〛 +〜 +〝 +〞 +〟 +〰 +– +— +‘ +’ +‛ +“ +” +„ +‟ +… +‧ +﹏ +< +> \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/upper_to_lower.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/upper_to_lower.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1b603c530f08b36c8a19a1110378491e3f22520e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/char/upper_to_lower.tsv @@ -0,0 +1,9 @@ +壹 一 +贰 二 +叁 三 +肆 四 +伍 五 +陆 六 +柒 七 +捌 八 +玖 九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/day.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/day.tsv new file mode 100644 index 0000000000000000000000000000000000000000..febc5762d6a778d5f023c0badba87a3f8a671f3f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/day.tsv @@ -0,0 +1,40 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +01 一 +02 二 +03 三 +04 四 +05 五 +06 六 +07 七 +08 八 +09 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/months.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d156409242c2a2ca8d670d53c3ee98717fb3a96f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/months.tsv @@ -0,0 +1,21 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +01 一 +02 二 +03 三 +04 四 +05 五 +06 六 +07 七 +08 八 +09 九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b620aa18107da918bd36342786d17a9a03b5a4e2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffix.tsv @@ -0,0 +1,16 @@ +ad 公元 +AD 公元 +a.d. 公元 +A.D. 公元 +ce 公元 +CE 公元 +c.e. 公元 +C.E. 公元 +bc 公元前 +BC 公元前 +b.c. 公元前 +B.C. 公元前 +bce 公元前 +BCE 公元前 +b.c.e. 公元前 +B.C.E. 公元前 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffixes.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffixes.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1370744b9147a76a39395cf7ea6634d41bc3c17a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/date/suffixes.tsv @@ -0,0 +1,16 @@ +ad 公元 +AD 公元 +a.d. 公元 +A.D. 公元 +ce 公元 +CE 公元 +c.e. 公元 +C.E. 公元 +bc 公元前 +BC 公元前 +b.c. 公元前 +B.C. 公元前 +bce 公元前 +BCE 公元前 +b.c.e. 公元前 +B.C.E. 公元前 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/denylist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/denylist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b419181407b9712f2e0ae41577957825486ca412 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/denylist/denylist.tsv @@ -0,0 +1,2 @@ +啊 +呃 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f00ae31a5bff5f9bf0ec59fde67f7cdb067850b9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/erhua/whitelist.tsv @@ -0,0 +1,35 @@ +儿女 +儿子 +儿孙 +女儿 +儿媳 +妻儿 +胎儿 +婴儿 +新生儿 +婴幼儿 +幼儿 +少儿 +小儿 +儿歌 +儿童 +儿科 +托儿所 +孤儿 +儿戏 +儿化 +台儿庄 +鹿儿岛 +正儿八经 +吊儿郎当 +生儿育女 +托儿带女 +养儿防老 +痴儿呆女 +佳儿佳妇 +儿怜兽扰 +儿无常父 +儿不嫌母丑 +儿行千里母担忧 +儿大不由爷 +苏乞儿 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/score.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/score.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0647248d5c49095e7f2be2773b84f457677fab71 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/score.tsv @@ -0,0 +1,2 @@ +: 比 +： 比 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4eff86d48b0f021427923724d2d7ddd8df592e15 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv @@ -0,0 +1,8 @@ ++ 加 +~ 到 +± 正负 += 等于 +× 乘 +÷ 除 +° 度 +- 减 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b1a8a832a67745b342ea8d43a29071c7d723c80f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv @@ -0,0 +1,67 @@ +amu 原子质量 +bar 巴 +°c 摄氏度 +°C 摄氏度 +ºc 摄氏度 +ºC 摄氏度 +℃ 摄氏度 +cm2 平方厘米 +cm² 平方厘米 +cm3 立方厘米 +cm³ 立方厘米 +cm 厘米 +cwt 美担 +db 分贝 +dm3 立方分米 +dm³ 立方分米 +dm 分米 +ds 毫秒 +°f 华氏度 +°F 华氏度 +℉ 华氏度 +ft 英尺 +ghz 吉赫兹 +gw 吉瓦 +gwh 吉瓦时 +hz 赫兹 +kbps 千比特每秒 +kcal 千卡 +kgf 千克力 +kg 千克 +khz 千赫兹 +km2 平方千米 +km² 平方千米 +km 千米 +kpa 千帕 +kwh 千瓦时 +kw 千瓦 +kW 千瓦 +lb 磅 +lbs 磅 +ng 纳克 +nm 纳米 +ns 纳秒 +oz 盎司 +pa 帕斯卡 +rad 弧度 +rpm 转每分 +sq ft 平方英尺 +sq mi 平方英里 +sv 系沃特 +tb 太字节 +tj 万亿焦耳 +tl 台两 +v 伏特 +yd 码 +μg 微克 +μm 微米 +μs 微秒 +ω 欧米茄 +gb 吉字节 +gpa 吉帕斯卡 +gy 戈瑞 +ha 公顷 +pg 皮克 +ps 皮秒 +s 秒 +g 克 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b8083350771c9bd85145181cd92bb8c2f01f2b8e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv @@ -0,0 +1,200 @@ +ALL 阿尔巴尼亚列克 +AFN 阿富汗 阿富汗尼 +ARS 阿根廷比索 +AWG 阿鲁巴盾 +AUD 澳元 +AZN 阿塞拜疆马纳特 +BSD 巴哈马元 +BBD 巴巴多斯元 +BYN 白俄罗斯卢布 +BZD 伯利兹元 +BMD 百慕大元 +BOB 玻利维亚玻利维亚诺 +BAM 波斯尼亚和黑塞哥维那可兑换马克 +BWP 博茨瓦纳普拉 +BGN 保加利亚列弗 +BRL 巴西雷亚尔 +BND 文莱达鲁萨兰国元 +KHR 柬埔寨瑞尔 +CAD 加元 +KYD 开曼群岛元 +CLP 智利比索 +CNY 人民币 +COP 哥伦比亚比索 +CRC 哥斯达黎加科隆 +HRK 克罗地亚库纳 +CUP 古巴比索 +CZK 捷克克朗 +DKK 丹麦克朗 +DOP 多米尼加共和国比索 +XCD 东加勒比元 +EGP 埃及镑 +SVC 萨尔瓦多科隆 +EUR 欧元成员国 +FKP 福克兰群岛（马尔维纳斯）镑 +FJD 斐济元 +GHS 加纳塞地 +GIP 直布罗陀镑 +GTQ 危地马拉格查尔 +GGP 根西岛镑 +GYD 圭亚那元 +HNL 洪都拉斯伦皮拉 +HKD 港元 +HUF 匈牙利福林 +ISK 冰岛克朗 +INR 印度卢比 +IDR 印尼盾 +IRR 伊朗里亚尔 +IMP 马恩岛英镑 +ILS 以色列谢克尔 +JMD 牙买加元 +JPY 日元 +JEP 泽西镑 +KZT 哈萨克斯坦腾格 +KPW 朝鲜园 +KRW 韩元 +KGS 吉尔吉斯斯坦索姆 +LAK 老挝基普 +LBP 黎巴嫩镑 +LRD 利比里亚元 +MKD 马其顿代纳尔 +MYR 马来西亚令吉 +MUR 毛里求斯卢比 +MXN 墨西哥比索 +MNT 蒙古图格里克 +MNT 摩洛哥迪拉姆 +MZN 莫桑比克梅蒂卡尔 +NAD 纳米比亚元 +NPR 尼泊尔卢比 +ANG 荷属安的列斯盾 +NZD 新西兰元 +NIO 尼加拉瓜科尔多瓦 +NGN 尼日利亚奈拉 +NOK 挪威克朗 +OMR 阿曼里亚尔 +PKR 巴基斯坦卢比 +PAB 巴拿马巴尔博亚 +PYG 巴拉圭瓜拉尼 +PEN 秘鲁索尔 +PHP 菲律宾比索 +PLN 波兰兹罗提 +QAR 卡塔尔里亚尔 +RON 罗马尼亚列伊 +RUB 俄罗斯卢布 +SHP 圣赫勒拿镑 +SAR 沙特阿拉伯里亚尔 +RSD 塞尔维亚第纳尔 +SCR 塞舌尔卢比 +SGD 新加坡元 +SBD 所罗门群岛元 +SOS 索马里先令 +ZAR 南非兰特 +LKR 斯里兰卡卢比 +SEK 瑞典克朗 +SRD 苏里南元 +SYP 叙利亚镑 +TWD 新台币 +THB 泰铢 +TTD 特立尼达和多巴哥元 +TRY 土耳其里拉 +TVD 图瓦卢元 +UAH 乌克兰格里夫纳 +AED 阿联酋迪拉姆 +GBP 英镑 +USD 美元 +UYU 乌拉圭比索 +UZS 乌兹别克斯坦索姆 +VEF 委内瑞拉玻利瓦尔 +VND 越南东 +YER 也门里亚尔 +ZWD 津巴布韦元 +J￥ 日元 +JPY￥ 日元 +HK＄ 港元 +A$ 澳元 +CAD$ 加元 +US$ 美元 +€ 欧元 +£ 英镑 +Fr 瑞士法郎 +₣ 法郎 +¥ 圆 +Kr 韩元 +NXN$ 墨西哥比索 +NZD$ 新西兰元 +SGD$ 新加坡元 +HKD$ 港元 +NOKKr 挪威克朗 +₩ 韩元 +TRY₺ 土耳其里拉 +₽ 俄罗斯卢布 +BRLR$ 巴西雷亚尔 +DKKKr 丹麦克朗 +TWDNT$ 新台币 +฿ 泰铢 +RM 马来西亚林吉特 +Rp 印尼盾 +Kč 捷克克朗 +₪ 以色列新谢克尔 +CLP$ 智利披索 +₱ 菲律宾披索 +د.إ 阿联酋迪拉姆 +COL$ 哥伦比亚披索 +L 罗马尼亚列伊 +JPY¥ 日元 +￥ 人民币 +Lek 阿尔巴尼亚列克 +ƒ 阿鲁巴盾 +Br 白俄罗斯卢布 +BZ$ 伯利兹元 +$b 玻利维亚玻利维亚诺 +KM 波斯尼亚和黑塞哥维那可兑换马克 +P 博茨瓦纳普拉 +лв 保加利亚列弗 +R$ 巴西雷亚尔 +៛ 柬埔寨瑞尔 +¥ 人民币 +₡ 哥斯达黎加科隆 +kn 克罗地亚库纳 +₱ 古巴比索 +kr 丹麦克朗 +RD$ 多米尼加共和国比索 +¢ 加纳塞地 +Q 危地马拉格查尔 +L 洪都拉斯伦皮拉 +Ft 匈牙利福林 +₹ 印度卢比 +￡ 英镑 +₪ 以色列谢克尔 +J$ 牙买加元 +лв 哈萨克斯坦腾格 +лв 吉尔吉斯斯坦索姆 +₭ 老挝基普 +ден 马其顿代纳尔 +₨ 毛里求斯卢比 +₮ 蒙古图格里克 +MT 莫桑比克梅蒂卡尔 +C$ 尼加拉瓜科尔多瓦 +₦ 尼日利亚奈拉 +₨ 巴基斯坦卢比 +B/. 巴拿马巴尔博亚 +Gs 巴拉圭瓜拉尼 +S/. 秘鲁索尔 +₱ 菲律宾比索 +zł 波兰兹罗提 +lei 罗马尼亚列伊 +₽ 卢布 +Дин. 塞尔维亚第纳尔 +S 索马里先令 +R 南非兰特 +CHF 瑞士法郎 +NT$ 新台币 +TT$ 特立尼达和多巴哥元 +₺ 土耳其里拉 +₴ 乌克兰格里夫纳 +$ 美元 +$U 乌拉圭比索 +лв 乌兹别克斯坦索姆 +Bs 委内瑞拉玻利瓦尔 +₫ 越南东 +Z$ 津巴布韦元 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_mandarin.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_mandarin.tsv new file mode 100644 index 0000000000000000000000000000000000000000..de1ccc7112245887075d796c9ae7c852b79066a4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/money/currency_mandarin.tsv @@ -0,0 +1,52 @@ +美元 +美金 +欧元 +英镑 +加元 +瑞士法郎 +法郎 +加拿大元 +元 +圆 +韩元 +墨西哥比索 +比索 +新西兰元 +新加坡元 +港元 +港币 +人民币 +挪威克朗 +克朗 +韩元 +土耳其里拉 +里拉 +印度卢比 +卢比 +俄罗斯卢布 +卢布 +巴西雷亚尔 +雷亚尔 +南非兰特 +兰特 +丹麦克朗 +波兰兹罗提 +罗提 +新台币 +台币 +泰铢 +马来西亚林吉特 +印尼盾 +盾 +匈牙利福林 +福林 +捷克克朗 +以色列新谢克尔 +新谢克尔 +智利比索 +菲律宾披索 +阿联酋迪拉姆 +迪拉姆 +哥伦比亚披索 +马来西亚令吉 +日元 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3a578d35897c0a09304357594a18a9ffa6e88b91 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit.tsv @@ -0,0 +1,9 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b949b95088a41bd4eef0cd97ac1dee6fb9b84ead --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv @@ -0,0 +1,9 @@ +1 一 +2 两 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_tens.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..884a8ba5fe724007ab175c9c97b184bb1bd116f9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/digit_tens.tsv @@ -0,0 +1,8 @@ +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/sign.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/sign.tsv new file mode 100644 index 0000000000000000000000000000000000000000..645adbdf19969bbf6b8a64d916de5a4d7674e0a6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/sign.tsv @@ -0,0 +1,2 @@ +- 负 ++ 正 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f44c0e151382c3066d4c15f487ab3e6168668082 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv @@ -0,0 +1,23 @@ +万 +十万 +百万 +千万 +亿 +十亿 +百亿 +千亿 +萬 +十萬 +百萬 +千萬 +億 +十億 +百億 +千億 +拾萬 +佰萬 +仟萬 +拾億 +佰億 +仟億 + diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..52dc01917bd4f329f0f49d48f0a2d7017354475f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen.tsv @@ -0,0 +1,10 @@ +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a48662621d42110aaec1fcf09cb1a01761981883 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv @@ -0,0 +1,10 @@ +10 一十 +11 一十一 +12 一十二 +13 一十三 +14 一十四 +15 一十五 +16 一十六 +17 一十七 +18 一十八 +19 一十九 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/ties.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2a73c039923ba4d315378a3c91b19b14e9c120e6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/ties.tsv @@ -0,0 +1,8 @@ +2 二十 +3 三十 +4 四十 +5 五十 +6 六十 +7 七十 +8 八十 +9 九十 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d6b9cece55a48a250d810fb13daa1917943ef90e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/number/zero.tsv @@ -0,0 +1 @@ +0 零 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/AM.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/AM.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b39eb5125c57212e430537e20f1f6573819a993e --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/AM.tsv @@ -0,0 +1,8 @@ +am +AM +a.m. +A.M. + am + AM + a.m. + A.M. \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/PM.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/PM.tsv new file mode 100644 index 0000000000000000000000000000000000000000..197fd1b3ab50c3c3e7aecad8013a781c49ddc576 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/PM.tsv @@ -0,0 +1,8 @@ +pm +p.m. +PM +P.M. + pm + p.m. + PM + P.M. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/hour.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/hour.tsv new file mode 100644 index 0000000000000000000000000000000000000000..17791c796d6ac3de0593200562ab01484c4a752a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/hour.tsv @@ -0,0 +1,35 @@ +1 一 +01 一 +2 两 +02 两 +3 三 +03 三 +4 四 +04 四 +5 五 +05 五 +6 六 +06 六 +7 七 +07 七 +8 八 +08 八 +9 九 +09 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +0 零 +00 零 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/minute.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/minute.tsv new file mode 100644 index 0000000000000000000000000000000000000000..68f849f4305b499ac53a96dc3df784d260295362 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/minute.tsv @@ -0,0 +1,71 @@ +1 一 +01 一 +2 二 +02 二 +3 三 +03 三 +4 四 +04 四 +5 五 +05 五 +6 六 +06 六 +7 七 +07 七 +8 八 +08 八 +9 九 +09 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十三 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 +0 零 +00 零 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/second.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/second.tsv new file mode 100644 index 0000000000000000000000000000000000000000..68f849f4305b499ac53a96dc3df784d260295362 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/second.tsv @@ -0,0 +1,71 @@ +1 一 +01 一 +2 二 +02 二 +3 三 +03 三 +4 四 +04 四 +5 五 +05 五 +6 六 +06 六 +7 七 +07 七 +8 八 +08 八 +9 九 +09 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十三 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 +0 零 +00 零 \ No newline at end of file diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/zero.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..573541ac9702dd3969c9bc859d2b91ec1f7e6e56 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/time/zero.tsv @@ -0,0 +1 @@ +0 diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/whitelist.tsv b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e8810f42ae838755e366cb2bee25c80c86543be4 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/data/whitelist.tsv @@ -0,0 +1,157 @@ +O 2 O O to O +O2O O to O +P 2 P P to P +P2P P to P +CD C D +a.m. AM +am. AM +A M AM +Am AM + am AM +p.m. PM +pm. PM +Pm PM + pm PM +atm ATM +A T M ATM +Atm ATM +ufo UFO +Ufo UFO +U F O UFO +u f o UFO +m v MV +mv MV +MV MV +F I F A fifa +f i f a fifa +FIFA fifa +nba NBA +Nba NBA +NBA NBA +n b a NBA +x L XL +X L XL +x x L XXL +X X L XXL +x x x L XXXL +X X X L XXXL +Q Q QQ +qq QQ +Qq QQ +gpu GPU +GPU GPU +cpu CPU +C P U CPU +WIFI wifi +ms MS +ceo CEO +c e o CEO +C E O CEO +dv DV +d v DV +D V DV +wto WTO +w t o WTO +W T O WTO +pc PC +p c PC +P C PC +gps GPS +G P S GPS +M.V.P MVP +mvp MVP +M V P MVP +vip VIP +V I P VIP +cba CBA +c b a CBA +CBA CBA +mba MBA +m b a MBA +M B A MBA +iq IQ +IQ IQ +EQ EQ +cctv CCTV +C C t v CCTV +kfc KFC +K F C KFC +Steam steam +O 2 O O to O +O2O O to O +P 2 P P to P +P2P P to P +CD C D +a.m. AM +am. AM +A M AM +Am AM + am AM +p.m. PM +pm. PM +Pm PM + pm PM +atm ATM +A T M ATM +Atm ATM +ufo UFO +Ufo UFO +U F O UFO +u f o UFO +m v MV +mv MV +MV MV +F I F A fifa +f i f a fifa +FIFA fifa +nba NBA +Nba NBA +NBA NBA +n b a NBA +x L XL +X L XL +x x L XXL +X X L XXL +x x x L XXXL +X X X L XXXL +Q Q QQ +qq QQ +Qq QQ +gpu GPU +GPU GPU +cpu CPU +C P U CPU +WIFI wifi +ms MS +ceo CEO +c e o CEO +C E O CEO +dv DV +d v DV +D V DV +wto WTO +w t o WTO +W T O WTO +pc PC +p c PC +P C PC +gps GPS +G P S GPS +M.V.P MVP +mvp MVP +M V P MVP +vip VIP +V I P VIP +cba CBA +c b a CBA +CBA CBA +mba MBA +m b a MBA +M B A MBA +iq IQ +IQ IQ +EQ EQ +cctv CCTV +C C t v CCTV +kfc KFC +K F C KFC diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/graph_utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c80c6e673ad9f39b19498f58f37cc17bd857bed2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/graph_utils.py @@ -0,0 +1,153 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.utils.logging import logger + +# ghaph_utils is here since importing from en folders will cause import errors +# that the data file names have to be the same with what are in the en folder + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_NOT_ALPHA = pynini.difference(NEMO_SIGMA, NEMO_ALPHA).optimize() +NEMO_SPACE_CHAR = pynini.union(NEMO_CHAR, NEMO_SPACE) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..caf02ccf13168c4b51da7d0eb29f479db39ab1f3 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/cardinal.py @@ -0,0 +1,192 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "二十三" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_digit_alt = pynini.string_file(get_abs_path("data/number/digit_alt.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/number/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/number/teen.tsv")) + graph_teen_alt = pynini.string_file(get_abs_path("data/number/teen_alt.tsv")) + + alls = NEMO_DIGIT**2 | NEMO_DIGIT**1 + graph_all = ( + (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen_alt | graph_digit + ) # graph_all when within a larger number e.g., 316-> 三百一十六 instead of 三百十六 + + graph_all = alls @ graph_all + graph_all_alt = ( + (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + ) # graph_all when at the head of the larger numbere.g., 13万 -> 十三万 instead of 一十三万 + graph_all_alt = alls @ graph_all_alt + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = (graph_digit + pynutil.insert('百')) + pynini.union( + pynini.closure(pynutil.delete('0')), + (pynini.closure(pynutil.delete('0') + pynutil.insert('零')) + graph_all), + ) + graph_hundred = hundreds @ graph_hundred_component + + self.digit = graph_digit.optimize() + self.all = graph_all.optimize() + + thousands = NEMO_DIGIT**4 + graph_thousand_component = (graph_digit_alt + pynutil.insert('千')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = (graph_digit_alt + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + + hundred_thousands = NEMO_DIGIT**6 + hundred_thousands_position = NEMO_DIGIT**2 + hundred_thousands_position = hundred_thousands_position @ graph_all_alt + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + million_position = NEMO_DIGIT**3 + million_position = million_position @ graph_hundred_component + graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + ten_million_position = NEMO_DIGIT**4 + ten_million_position = ten_million_position @ graph_thousand_component + graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit_alt + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + (pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + thousand_millions_position = NEMO_DIGIT**2 + thousand_millions_position = thousand_millions_position @ graph_all_alt + graph_thousand_million_component = (thousand_millions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + ten_billions = NEMO_DIGIT**11 + ten_billions_position = NEMO_DIGIT**3 + ten_billions_position = ten_billions_position @ graph_hundred_component + graph_ten_billions_component = (ten_billions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**12 + hundred_billions_position = NEMO_DIGIT**4 + hundred_billions_position = hundred_billions_position @ graph_thousand_component + graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + graph = pynini.union( + graph_hundred_billions, + graph_ten_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_all_alt, + graph_zero, + ) + self.just_cardinals = graph.optimize() + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("负", "-")) + pynutil.insert("\"") + ) + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + self.with_sign = final_graph.optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..5d20aff48cc490c4b278804ad962bb81478875db --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/date.py @@ -0,0 +1,128 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classfying dates, e.g. + 2002年 -> tokens { date { year: "二零零二" } } + 2002-01-28 -> tokens { date { year: "二零零二" month: "一" day: "二十八"} } + 2002/01/28 -> tokens { date { year: "二零零二" month: "一" day: "二十八"} } + 2002.01.28 -> tokens { date { year: "二零零二" month: "一" day: "二十八"} } + 2002年2月 -> tokens { date { year: "二零零二" month: "二" } } + 2月11日 -> tokens { date { month: "二" day: "十一" } } + 2002/02 -> is an error format according to the national standard + 02/11 -> is an error format according to the national standard + According to national standard, only when the year, month, and day are all exist, it is allowed to use symbols to separate them + + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + month = pynini.string_file(get_abs_path("data/date/months.tsv")) + day = pynini.string_file(get_abs_path("data/date/day.tsv")) + suffix = pynini.string_file(get_abs_path("data/date/suffixes.tsv")) + + delete_sign = pynutil.delete('/') | pynutil.delete('-') | pynutil.delete('.') | pynutil.delete('·') + delete_day = pynutil.delete('号') | pynutil.delete('號') | pynutil.delete('日') + + # grammar for only year, month, or day + # atleast accep two digit to distinguish from year used for time + # don't accept 日 to distinguish from day used fro time + only_year = ( + pynutil.insert("year: \"") + + pynini.closure(graph_digit | graph_zero, 2) + + pynutil.delete('年') + + pynutil.insert("\"") + ) + only_month = pynutil.insert("month: \"") + month + pynutil.delete('月') + pynutil.insert("\"") + only_day = pynutil.insert("day: \"") + day + delete_day + pynutil.insert("\"") + # gh_1 + graph_only_date = only_year | only_month | only_day + + year_month = only_year + pynutil.insert(' ') + only_month + month_day = only_month + pynutil.insert(' ') + only_day + graph_ymd = only_year + pynutil.insert(' ') + only_month + pynutil.insert(' ') + only_day + # gh_2 + graph_combination = year_month | month_day | graph_ymd + + year_component = ( + pynutil.insert("year: \"") + + pynini.closure(graph_digit | graph_zero, 2, 4) + + delete_sign + + pynutil.insert("\"") + ) + month_component = pynutil.insert("month: \"") + month + delete_sign + pynutil.insert("\"") + day_component = pynutil.insert("day: \"") + day + pynutil.insert("\"") + # gp_3 + graph_sign = year_component + pynutil.insert(' ') + month_component + pynutil.insert(' ') + day_component + # gp_1+2+3 + graph_all = graph_only_date | graph_sign | graph_combination + + prefix = ( + pynini.accep('公元') + | pynini.accep('西元') + | pynini.accep('公元前') + | pynini.accep('西元前') + | pynini.accep('纪元') + | pynini.accep('纪元前') + ) + prefix_component = pynutil.insert("era: \"") + prefix + pynutil.insert("\"") + # gp_prefix+(1,2,3) + graph_prefix = prefix_component + pynutil.insert(' ') + (graph_ymd | year_month | only_year) + + suffix_component = pynutil.insert("era: \"") + suffix + pynutil.insert("\"") + # gp_suffix +(1,2,3) + graph_suffix = (graph_ymd | year_month | only_year) + pynutil.insert(' ') + suffix_component + # gp_4 + graph_affix = graph_prefix | graph_suffix + + graph_suffix_year = ( + pynutil.insert("year: \"") + pynini.closure((graph_digit | graph_zero), 1) + pynutil.insert("\"") + ) + graph_suffix_year = graph_suffix_year + pynutil.insert(' ') + suffix_component + + graph_with_era = graph_suffix_year | graph_affix + + graph = graph_only_date | graph_combination | graph_sign | graph_with_era + + # range + symbol = pynini.accep("-") | pynini.accep("~") | pynini.accep("——") | pynini.accep("—") + range_source = pynutil.insert("range: \"") + pynini.closure("从", 0, 1) + pynutil.insert("\"") + range_goal = ( + pynutil.insert("range: \"") + + (pynini.closure("到", 0, 1) | pynini.closure("至", 0, 1) | symbol) + + pynutil.insert("\"") + ) + graph_source = ( + range_source + pynutil.insert(' ') + graph + pynutil.insert(' ') + range_goal + pynutil.insert(' ') + graph + ) + graph_goal = graph + pynutil.insert(' ') + range_goal + pynutil.insert(' ') + graph + + graph_range_final = graph_source | graph_goal + + final_graph = pynutil.add_weight(graph, -2.0) | graph_range_final + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..38106984c11037788d6f685f8cfb4852d59c2b16 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/decimal.py @@ -0,0 +1,115 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +def get_quantity(decimal): + suffix = pynini.union( + "万", + "十万", + "百万", + "千万", + "亿", + "十亿", + "百亿", + "千亿", + "萬", + "十萬", + "百萬", + "千萬", + "億", + "十億", + "百億", + "千億", + "拾萬", + "佰萬", + "仟萬", + "拾億", + "佰億", + "仟億", + "拾万", + "佰万", + "仟万", + "仟亿", + "佰亿", + "仟亿", + "万亿", + "萬億", + ) + res = decimal + pynutil.insert(" quantity: \"") + suffix + pynutil.insert("\"") + + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 0.5 -> decimal { integer_part: "零" fractional_part: "五" } + 0.5万 -> decimal { integer_part: "零" fractional_part: "五" quantity: "万" } + -0.5万 -> decimal { negative: "负" integer_part: "零" fractional_part: "五" quantity: "万"} + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_before_decimal = cardinal.just_cardinals + cardinal_after_decimal = pynini.string_file(get_abs_path("data/number/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") + + graph_fraction = ( + pynutil.insert("fractional_part: \"") + + pynini.closure((pynini.closure(cardinal_after_decimal, 1) | (pynini.closure(zero, 1))), 1) + + pynutil.insert("\"") + ) + graph_decimal = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction + self.regular_decimal = graph_decimal.optimize() + + graph_sign = ( + ( + pynini.closure(pynutil.insert("negative: \"") + pynini.cross("-", "负")) + + pynutil.insert("\"") + + pynutil.insert(" ") + ) + ) | ( + ( + pynutil.insert('negative: ') + + pynutil.insert("\"") + + (pynini.accep('负') | pynini.cross('負', '负')) + + pynutil.insert("\"") + + pynutil.insert(' ') + ) + ) + graph_with_sign = graph_sign + graph_decimal + graph_regular = graph_with_sign | graph_decimal + + graph_decimal_quantity = get_quantity(graph_decimal) + graph_sign_quantity = graph_sign + graph_decimal_quantity + graph_quantity = graph_decimal_quantity | graph_sign_quantity + + final_graph = graph_regular | graph_quantity + self.decimal = final_graph.optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..9ab37e6fbf7fccb8512977e7e68fab3b71f4341a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/fraction.py @@ -0,0 +1,167 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction, e.g., + 1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } + 5又1/2 -> tokens { fraction { integer_part: "五" denominator: "二" numerator: "一" } } + 5又2分之1 -> tokens { {} } + 2分之1 -> tokens { fraction { denominator: "二" numerator: "一"} } + 100分之1 -> tokens { fraction { denominator: "一百" numerator: "一"} } + 百分之1 -> tokens { fraction { denominator: "百" numerator: "一"} } + 98% -> tokens { fraction { denominator: "百" numerator: "九十八"} } + + Args: + cardinal: CardinalFst, decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + graph_cardinals = cardinal.just_cardinals + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + slash = pynutil.delete('/') + morpheme = pynutil.delete('分之') + suffix = pynini.union( + "百", + "千", + "万", + "十万", + "百万", + "千万", + "亿", + "十亿", + "百亿", + "千亿", + "萬", + "十萬", + "百萬", + "千萬", + "億", + "十億", + "百億", + "千億", + "拾萬", + "佰萬", + "仟萬", + "拾億", + "佰億", + "仟億", + "拾万", + "佰万", + "仟万", + "仟亿", + "佰亿", + "仟亿", + ) + + integer_component = pynutil.insert('integer_part: \"') + graph_cardinals + pynutil.insert("\"") + denominator_component = pynutil.insert("denominator: \"") + graph_cardinals + pynutil.insert("\"") + numerator_component = pynutil.insert("numerator: \"") + graph_cardinals + pynutil.insert("\"") + + graph_with_integer = ( + pynini.closure(integer_component + pynutil.delete('又'), 0, 1) + + pynutil.insert(' ') + + numerator_component + + slash + + pynutil.insert(' ') + + denominator_component + ) # 5又1/3 + + graph_only_slash = numerator_component + slash + pynutil.insert(' ') + denominator_component + + graph_morpheme = (denominator_component + morpheme + pynutil.insert(' ') + numerator_component) | ( + integer_component + + pynutil.delete('又') + + pynutil.insert(' ') + + denominator_component + + morpheme + + pynutil.insert(' ') + + numerator_component + ) # 5又3分之1 + + graph_with_suffix = ( + pynini.closure(pynutil.insert("denominator: \"") + suffix + pynutil.insert("\""), 0, 1) + + morpheme + + pynutil.insert(' ') + + numerator_component + ) # 万分之1 + + percentage = pynutil.delete('%') + + graph_decimal = ( + pynutil.insert('integer_part: \"') + + pynini.closure( + graph_cardinals + + pynutil.delete('.') + + pynutil.insert('点') + + pynini.closure((graph_digit | graph_zero), 1) + ) + + pynutil.insert("\"") + ) + graph_decimal_percentage = pynini.closure( + graph_decimal + percentage + pynutil.insert(' denominator: \"百"'), 1 + ) # 5.6% + + graph_integer_percentage = pynini.closure( + (numerator_component) + percentage + pynutil.insert(' denominator: \"百"'), 1 + ) # 5% + + graph_hundred = pynutil.delete('100%') + pynutil.insert('numerator: \"百\" denominator: \"百"') + # 100% + + graph_optional_sign = (pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"负\""))) | ( + pynutil.insert('negative: ') + + pynutil.insert("\"") + + (pynini.accep('负') | pynini.cross('負', '负')) + + pynutil.insert("\"") + ) + + graph = pynini.union( + graph_with_integer, + graph_only_slash, + graph_morpheme, + graph_with_suffix, + graph_decimal_percentage, + graph_integer_percentage, + graph_hundred, + ) + graph_with_sign = ( + (graph_optional_sign + pynutil.insert(" ") + graph_with_integer) + | (graph_optional_sign + pynutil.insert(" ") + graph_only_slash) + | (graph_optional_sign + pynutil.insert(" ") + graph_morpheme) + | (graph_optional_sign + pynutil.insert(" ") + graph_with_suffix) + | (graph_optional_sign + pynutil.insert(" ") + graph_integer_percentage) + | (graph_optional_sign + pynutil.insert(" ") + graph_decimal_percentage) + | (graph_optional_sign + pynutil.insert(" ") + graph_hundred) + ) + + final_graph = graph | pynutil.add_weight(graph_with_sign, -3.0) + + self.just_fractions = graph.optimize() + self.fractions = final_graph.optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..5d9c5f58335e2aae876e752ac93b3b00a7ad79af --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/measure.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, insert_space +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class MeasureFst(GraphFst): + ''' + 1kg -> tokens { measure { cardinal { integer: "一" } units: "千克" } } + ''' + + def __init__( + self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True, lm: bool = False + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + units_en = pynini.string_file(get_abs_path("data/measure/units_en.tsv")) + + graph_cardinal = cardinal.with_sign + graph_decimal = decimal.decimal + + # these units ared added due to falures when running Sparrow Hawk tests that "ms" would be processed as "m" and "s" left outside of the tagegr + units = ( + pynini.cross("ms", "毫秒") + | pynini.cross("m²", "平方米") + | pynini.cross("m2", "平方米") + | pynini.cross("m²", "平方米") + | pynini.cross("m³", "立方米") + | pynini.cross("mbps", "兆比特每秒") + | pynini.cross("mg", "毫克") + | pynini.cross("mhz", "兆赫兹") + | pynini.cross("mi2", "平方英里") + | pynini.cross("mi²", "平方英里") + | pynini.cross("mi", "英里") + | pynini.cross("min", "分钟") + | pynini.cross("ml", "毫升") + | pynini.cross("mm2", "平方毫米") + | pynini.cross("mm²", "平方毫米") + | pynini.cross("mol", "摩尔") + | pynini.cross("mpa", "兆帕") + | pynini.cross("mph", "英里每小时") + | pynini.cross("mm", "毫米") + | pynini.cross("mv", "毫伏") + | pynini.cross("mw", "毫瓦") + ) + + unit_component = pynutil.insert("units: \"") + (units_en | units) + pynutil.insert("\"") + + graph_cardinal_measure = pynini.closure( + (pynutil.insert("cardinal { ") + graph_cardinal + pynutil.insert(" } ") + insert_space + unit_component), 1 + ) + + graph_decimal_measure = pynini.closure( + (pynutil.insert("decimal { ") + graph_decimal + pynutil.insert(" } ") + unit_component), 1 + ) + + graph_measures = graph_decimal_measure | graph_cardinal_measure + + final_graph = self.add_tokens(graph_measures) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..6012bd7bcd9aa29d706c11159a2710b834580cfd --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/money.py @@ -0,0 +1,139 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + +suffix = pynini.union( + "万", + "十万", + "百万", + "千万", + "亿", + "十亿", + "百亿", + "千亿", + "萬", + "十萬", + "百萬", + "千萬", + "億", + "十億", + "百億", + "千億", + "拾萬", + "佰萬", + "仟萬", + "拾億", + "佰億", + "仟億", + "拾万", + "佰万", + "仟万", + "仟亿", + "佰亿", + "仟亿", + "万亿", + "萬億", +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying moneys, e.g. + '$23' -> money { integer: "二十三" currency: "美元" } + '23美元' -> money { integer: "二十三" currency: "美元" } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + cardinal = cardinal.just_cardinals + + currency = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + currency_mandarin = pynini.string_file(get_abs_path("data/money/currency_mandarin.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + # regular money gramamr with currency symbols $1000 + currency_component = pynutil.insert("currency: \"") + currency + pynutil.insert("\"") + number_component = pynutil.insert("integer_part: \"") + (cardinal | (cardinal + suffix)) + pynutil.insert("\"") + graph_regular_money = currency_component + pynutil.insert(" ") + number_component + + # 块 元 毛 with optional symbols + unit_major = ( + pynini.accep("块") + | pynini.accep("元") + | pynini.closure(pynini.cross("塊", "块"), 1) + | pynini.closure(pynini.cross("塊錢", "块钱"), 1) + | pynini.accep("块钱") + ) + unit_minor = pynini.accep("角") | pynini.accep("毛") + unit_minor_alt = pynini.accep("分") + currency_mandarin_component = pynutil.insert("currency: \"") + currency_mandarin + pynutil.insert("\"") + unit_components = ( + (pynutil.insert("currency: \"") + unit_major + pynutil.insert("\"")) + | (pynutil.insert("currency_maj: \"") + unit_minor + pynutil.insert("\"")) + | (pynutil.insert("currency_min: \"") + unit_minor_alt + pynutil.insert("\"")) + ) + + graph_unit_only = ( + number_component + + pynutil.insert(" ") + + unit_components + + pynini.closure(pynutil.insert(" ") + currency_mandarin_component, 0, 1) + ) + graph_units = pynini.closure(graph_unit_only, 1, 3) + + # only currency part as mandarins + graph_mandarin_money = number_component + pynutil.insert(" ") + currency_mandarin_component + + # larger money as decimals + graph_decimal = ( + pynutil.insert('integer_part: \"') + + ( + pynini.closure(cardinal, 1) + + pynutil.delete('.') + + pynutil.insert('点') + + pynini.closure((graph_digit | graph_zero), 1) + ) + + pynutil.insert("\"") + ) + graph_decimal_money = ( + pynini.closure(graph_decimal, 1) + + pynini.closure((pynutil.insert(' quantity: \"') + suffix + pynutil.insert('\"')), 0, 1) + + pynutil.insert(" ") + + pynini.closure(currency_mandarin_component, 1) + ) | ( + pynini.closure(currency_component, 1) + + pynutil.insert(" ") + + pynini.closure(graph_decimal, 1) + + pynini.closure( + (pynutil.insert(" ") + pynutil.insert('quantity: \"') + suffix + pynutil.insert('\"')), 0, 1 + ) + ) + + graph = ( + graph_regular_money + | graph_units + | pynutil.add_weight(graph_mandarin_money, -3.0) + | pynutil.add_weight(graph_decimal_money, -1.0) + ) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..beb73fe79032c1cb76a72e59c05f6295bfacc669 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/ordinal.py @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 第100 -> ordinal { integer: "第一百" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + morpheme = pynini.accep('第') + graph_ordinal = morpheme + graph_cardinal + graph_ordinal_final = pynutil.insert("integer: \"") + graph_ordinal + pynutil.insert("\"") + + # range + range_source = pynini.accep("从") + range_goal = ( + pynini.accep("-") + | pynini.accep("~") + | pynini.accep("——") + | pynini.accep("—") + | pynini.accep("到") + | pynini.accep("至") + ) + graph_range_source = ( + pynini.closure((pynutil.insert("range: \"") + range_source + pynutil.insert("\" ")), 0, 1) + + pynutil.insert("integer: \"") + + graph_ordinal + + pynutil.insert("\"") + + pynutil.insert(" range: \"") + + range_goal + + pynutil.insert("\" ") + + pynutil.insert("integer: \"") + + (graph_ordinal | graph_cardinal) + + pynutil.insert("\"") + ) + graph_range_goal = ( + pynutil.insert("integer: \"") + + graph_ordinal + + pynutil.insert("\"") + + pynutil.insert(" range: \"") + + range_goal + + pynutil.insert("\" ") + + pynutil.insert("integer: \"") + + (graph_ordinal | graph_cardinal) + + pynutil.insert("\"") + ) + graph_range_final = graph_range_source | graph_range_goal + + final_graph = graph_ordinal_final | graph_range_final + + graph_ordinal_final = self.add_tokens(final_graph) + self.fst = graph_ordinal_final.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..1953fc1853f0a4eb6f38f4cd74cd2e17b7fe28a9 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class PreProcessorFst(GraphFst): + ''' + Preprocessing of TN: + 1. interjections removal such as '啊, 呃' + 2. fullwidth -> halfwidth char conversion + 好啊 -> 好 + 呃对 -> 对 + ： -> : + ； -> ; + ''' + + def __init__( + self, + remove_interjections: bool = True, + fullwidth_to_halfwidth: bool = True, + ): + super().__init__(name="PreProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + + if remove_interjections: + remove_interjections_graph = pynutil.delete(pynini.string_file(get_abs_path('data/denylist/denylist.tsv'))) + graph @= pynini.cdrewrite(remove_interjections_graph, '', '', NEMO_SIGMA) + + if fullwidth_to_halfwidth: + fullwidth_to_halfwidth_graph = pynini.string_file(get_abs_path('data/char/fullwidth_to_halfwidth.tsv')) + graph @= pynini.cdrewrite(fullwidth_to_halfwidth_graph, '', '', NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/punctuation.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..a3aa062375a5722f4b46e93e4f3ccee9e4b89618 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/punctuation.py @@ -0,0 +1,67 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-/:;<=>?@^_`{|}~\"。，；-《》“”" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + whitelist_symbols = load_labels(get_abs_path("data/whitelist/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..26e9f94abdf95cfbdfe77ef022fe3d968fdec373 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/time.py @@ -0,0 +1,149 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for clasifying time, e.g. + 01:02 -> tokens { time { hours: "一" minutes: "二" } } + 1:02:03 -> tokens { time { hours: "一" minutes: "二" second: "三" } } + 1点5分19秒 -> tokens { time { hours: "一" minutes: "五" second: "秒" } } + 1点1刻 -> tokens { time { hours: "一" minutes: "一刻" } } + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + # mappings imported + hour = pynini.string_file(get_abs_path("data/time/hour.tsv")) + minute = pynini.string_file(get_abs_path("data/time/minute.tsv")) + second = pynini.string_file(get_abs_path("data/time/second.tsv")) + alphabet_am = pynini.string_file(get_abs_path("data/time/AM.tsv")) + alphabet_pm = pynini.string_file(get_abs_path("data/time/PM.tsv")) + + # gramamr for time, separated by colons 05:03:13 + symbol = pynutil.delete(":") | pynutil.delete("：") + hour_component = pynutil.insert("hours: \"") + hour + pynutil.insert('点') + pynutil.insert("\"") + minute_component = pynutil.insert("minutes: \"") + minute + pynutil.insert('分') + pynutil.insert("\"") + second_component = pynutil.insert("seconds: \"") + second + pynutil.insert('秒') + pynutil.insert("\"") + # combining 3 components + hour_minute_second = ( + hour_component + + symbol + + pynutil.insert(' ') + + minute_component + + symbol + + pynutil.insert(' ') + + second_component + ) + hour_minute = hour_component + symbol + pynutil.insert(' ') + minute_component + graph_colon = hour_minute_second | hour_minute + + # gramamr for time as clock, with morphems, 点， 分， 秒 + hour_clock = pynini.accep("点") | pynini.cross("點", "点") + minute_clock = pynini.accep("分") | pynini.accep('刻') + second_clock = pynini.accep('秒') + # grammar for time, as period of time 小时，分钟，秒 + hour_duration = ( + pynini.accep('个点') + | pynini.cross("個點", "個点") + | pynini.accep('小时') + | pynini.cross('小時', '小时') + | pynini.accep('个小时') + | pynini.cross('個小時', '个小时') + | pynini.accep('个钟头') + | pynini.cross('個鐘頭', '个钟头') + ) + minute_duration = pynini.accep("分钟") | pynini.accep('刻') | pynini.accep('刻钟') + second_duration = pynini.accep("秒钟") | pynini.cross('秒鐘', '秒钟') | pynini.accep('秒') + # combining two above + hour_component = pynutil.insert("hours: \"") + hour + (hour_clock | hour_duration) + pynutil.insert("\"") + minute_component = ( + pynutil.insert("minutes: \"") + minute + (minute_clock | minute_duration) + pynutil.insert("\"") + ) + second_component = ( + pynutil.insert("seconds: \"") + second + (second_clock | second_duration) + pynutil.insert("\"") + ) + hour_minute = hour_component + pynutil.insert(' ') + minute_component + hour_second = hour_component + pynutil.insert(' ') + second_component + minute_second = minute_component + pynutil.insert(' ') + second_component + clock_all = hour_component + pynutil.insert(' ') + minute_component + pynutil.insert(' ') + second_component + graph_clock_period = ( + hour_component + | minute_component + | second_component + | hour_minute + | hour_second + | minute_second + | clock_all + ) + + # gramamr for time, back count; 五点差n分n秒 + backcount = pynutil.insert("morphosyntactic_features: \"") + pynini.accep('差') + pynutil.insert("\"") + graph_hour = ( + ( + pynini.closure(backcount) + + pynutil.insert(' ') + + hour_component + + pynutil.insert(' ') + + pynini.closure(backcount) + + pynutil.insert(' ') + + minute_component + ) + | ( + pynini.closure(backcount) + + pynutil.insert(' ') + + hour_component + + pynutil.insert(' ') + + pynini.closure(backcount) + + pynutil.insert(' ') + + second_component + ) + | ( + pynini.closure(backcount) + + pynutil.insert(' ') + + hour_component + + pynutil.insert(' ') + + pynini.closure(backcount) + + pynutil.insert(' ') + + minute_component + + pynutil.insert(' ') + + second_component + ) + ) + graph_minute = minute_component + pynutil.insert(' ') + backcount + pynutil.insert(' ') + second_component + graph_backcount = graph_hour | graph_minute + + # grammar for time, with am, pr, or Mandarin words as prefiex/suffix 早上5点 05：04：04am + suffix_am = pynini.closure(alphabet_am, 0, 1) + suffix_pm = pynini.closure(alphabet_pm, 0, 1) + + am_component = pynutil.insert("suffix: \"") + suffix_am + pynutil.insert("\"") + pm_component = pynutil.insert("suffix: \"") + suffix_pm + pynutil.insert("\"") + + graph_suffix = ( + (graph_clock_period | graph_colon | graph_backcount) + pynutil.insert(' ') + (am_component | pm_component) + ) + + graph = graph_colon | graph_clock_period | graph_backcount | graph_suffix + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..2171e55adcd06b84ab8c432d188f53048b481e19 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py @@ -0,0 +1,100 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, generator_main +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.punctuation import PunctuationFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.whitelist import WhiteListFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + else: + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(deterministic=deterministic) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.05), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(word.fst, 100), + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") + tagger = pynini.closure(token, 1) + + self.fst = tagger + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..ad93b0c0f88b943eaab64dcb0c9005e157d44397 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/whitelist.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, convert_space +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "Mr." -> tokens { name: "mister" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(file): + whitelist = load_labels(file) + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(get_abs_path("data/whitelist.tsv")) + + graph |= pynutil.add_weight(_get_whitelist_graph(get_abs_path("data/whitelist.tsv")), weight=0.0001) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e87281ba825a819616c8a19c64850b9cc5a9ca --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. dormir -> tokens { name: "dormir" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + NEMO_NOT_QUOTE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/utils.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..175aba20613e6a63c715114c1b37ffcd202082eb --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/utils.py @@ -0,0 +1,66 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gaph_utils is kept due to the fatc that importing from en folders will cause import errors that the data file names have to be the same with what are in the en folder +import csv +import os + +UNIT_1e01 = '十' +UNIT_1e02 = '百' +UNIT_1e03 = '千' +UNIT_1e04 = '万' + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/__init__.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..edbc5d0aec449f01329e5807874e45f17dbd936f --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "负" integer: "23" } -> 负二十三 + cardinal { integer: "23" } -> 二十三 + cardinal { positive: "正" integer: "23" } -> 正二十三 + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + delete_sign = pynini.cross("negative: \"-\"", "负") + delete_integer = ( + pynutil.delete("integer: ") + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + delete_mandarin = pynutil.delete("quantity: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph_mandarin = (delete_integer + delete_mandarin) | ( + delete_sign + delete_space + delete_integer + delete_mandarin + ) + graph_sign = delete_sign + delete_space + delete_integer + final_graph = delete_integer | graph_sign | graph_mandarin + self.numbers = final_graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/date.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..8f0c1f231d27f9e5710b1530ea42839ae0460914 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/date.py @@ -0,0 +1,105 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DateFst(GraphFst): + """ + Finite state transducer classifiying dates, e.g. + { date { year: "二零零二" } } -> 二零零二年 + { date { year: "二零零二" month: "一" day: "二十八"} } -> 二零零二年一月二十八日 + { date { year: "二零零二" month: "二" } } -> 二零零二年二月 + { date { month: "二" day: "十一" } } -> 二月十一日 + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + year_component = ( + pynutil.delete("year: ") + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + pynutil.insert("年") + ) + month_component = ( + pynutil.delete("month: ") + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + pynutil.insert("月") + ) + day_component = ( + pynutil.delete("day: ") + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + pynutil.insert("日") + ) + + optional_era = ( + pynutil.delete("era: ") + pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + ) + + graph_date = ( + pynini.closure(year_component) + + pynini.closure(delete_space) + + pynini.closure(month_component) + + pynini.closure(delete_space) + + pynini.closure(day_component) + ) + + graph_date_era = pynini.union( + (optional_era + delete_space + year_component), + (optional_era + delete_space + year_component + delete_space + month_component), + ( + optional_era + + delete_space + + year_component + + delete_space + + month_component + + delete_space + + day_component + ), + ) + + graph_date_all = graph_date | graph_date_era + + # range + symbol = pynini.accep("-") | pynini.accep("~") | pynini.accep("——") | pynini.accep("—") + ranges = ( + pynutil.delete("range: \"") + + delete_space + + (pynini.closure(NEMO_NOT_QUOTE) | pynini.cross(symbol, "到")) + + pynutil.delete("\"") + ) + graph_range = ( + pynini.closure((ranges + delete_space), 0, 1) + + graph_date + + delete_space + + ranges + + delete_space + + graph_date + ) + + final_graph = graph_date_all | graph_range + # final_graph = optional_era + delete_space + year_component + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..6158d56f9aa7c6c67308596d7c0aa56b77484397 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py @@ -0,0 +1,64 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { integer_part: "零" fractional_part: "五" } -> 零点五 + decimal { integer_part: "零" fractional_part: "五" quantity: "万" } -> 零点五万 + decimal { positive: "正" integer_part: "零" fractional_part: "五" quantity: "万" } -> 正零点五万 + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + fractional = pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.accep("负") + + pynutil.delete("\"") + ) + + graph = integer + delete_space + pynutil.insert("点") + fractional + self.decimal_regular = graph + graph_quantity = graph + delete_space + quantity + graph_regular = graph | graph_quantity + + graph_sign = sign + delete_space + graph_regular + + final_graph = graph_regular | graph_sign + self.decimal_component = final_graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..0eeffc06d60c3df380aa663a7a6baad723402826 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction e.g. + tokens { fraction { denominator: "二" numerator: "一"} } -> 二分之一 + tokens { fraction { integer_part: "一" denominator: "二" numerator: "一" } } -> 一又二分之一 + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + graph_decimal = decimal.decimal_component + + integer_part = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.insert("又") + + pynutil.delete("\"") + ) + denominator_part = ( + pynutil.delete("denominator:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + numerator_part = ( + pynutil.delete("numerator:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + sign_part = ( + pynutil.delete("positive:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) | ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + graph_with_integer = ( + integer_part + delete_space + denominator_part + delete_space + pynutil.insert('分之') + numerator_part + ) + graph_no_integer = denominator_part + delete_space + pynutil.insert('分之') + numerator_part + graph = graph_with_integer | graph_no_integer + + graph_with_decimal = ( + denominator_part + + delete_space + + pynutil.insert('分之') + + pynutil.delete("integer_part: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + graph_with_sign = sign_part + delete_space + (graph | graph_with_decimal) + + final_graph = graph_with_sign | graph | graph_with_decimal + self.fraction = final_graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/measure.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..9e964ae80cc076d8fded88f6d390d906c9c5d7c2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/measure.py @@ -0,0 +1,71 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class MeasureFst(GraphFst): + ''' + tokens { measure { cardinal: "一" } units: "千克" } } -> 一千克 + ''' + + def __init__( + self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True, lm: bool = False + ): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + cardinal = cardinal.numbers + decimal = decimal.decimal_component + sign_component = pynutil.delete("negative: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + unit_component = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_cardinal = ( + pynutil.delete("cardinal { ") + cardinal + pynutil.delete(" } ") + delete_space + unit_component + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + decimal + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + pynutil.delete("}") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + delete_space + + unit_component + ) + + graph_fraction = ( + pynutil.delete("fraction {") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + fraction.fraction + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + pynutil.delete("}") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + delete_space + + unit_component + ) + + graph_math_cardinal = pynutil.delete("cardinal { ") + cardinal + pynutil.delete(" } ") + + graph_measures = graph_decimal | graph_cardinal | graph_fraction + graph_maths = graph_math_cardinal + + final_graph = graph_maths | graph_measures + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/money.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..a0866aca65b03080a5ad11f2ce8d0705428bec00 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/money.py @@ -0,0 +1,65 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing fraction e.g. + tokens { money { integer: "二" currency: "$"} } -> 二美元 + tokens { money { integer: "三" major_unit: "块"} } -> 三块 + tokens { money { currency: "$" integer: "二" } } -> 二美元 + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + # components to combine to make graphs + number_component = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + currency_component = pynutil.delete("currency: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + decimal_component = decimal.decimal_component + unit_only_component = ( + (pynutil.delete("currency: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) + | (pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) + | (pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) + ) + + # graphs + graph_regular_money = number_component + delete_space + currency_component + graph_unit_money = pynini.closure( + (number_component + delete_space + unit_only_component + pynini.closure(delete_space)) + ) + graph_decimal_money = decimal_component + delete_space + currency_component + + graph_suffix = ( + number_component + + delete_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + + currency_component + ) + + graph = graph_unit_money | graph_regular_money | graph_decimal_money | graph_suffix + + final_graph = graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..81a3cab37188c5a8a1e16d7495c2f34b74624e64 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal e.g. + tokens { ordinal { integer: "第一千万" } } -> 第一千万 + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + symbol = pynini.union("-", "~", "——", "—") + dash = pynini.cross(symbol, "到") + + delete_morpheme = ( + pynutil.delete("range: \"") + (pynini.closure('从') | (pynini.closure('到') | dash)) + pynutil.delete("\"") + ) + graph_integer = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure("第", 0, 1) + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + graph_range = ( + pynini.closure(delete_morpheme, 0, 1) + + pynini.closure(delete_space, 0, 1) + + graph_integer + + delete_space + + delete_morpheme + + delete_space + + graph_integer + ) + + final_graph = pynutil.add_weight(graph_integer, -2.0) | graph_range + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..51f3e8737fa81cccc66212027acf5e5e31a1e5cf --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini + +from zonos2.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_SPACE, + NEMO_SIGMA, + delete_space, + generator_main, +) +from zonos2.vendor.nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + + remove_space_around_single_quote = pynini.cdrewrite( + delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + ) + # this works if spaces in between (good) + # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA + + graph = remove_space_around_single_quote.optimize() + + return graph diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..c47f603341d185c3e24444f35586c4fa58c7b78a --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py @@ -0,0 +1,81 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil, utf8 + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_PUNCT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, +) +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class PostProcessor(GraphFst): + ''' + Postprocessing of TN, now contains: + 1. punctuation removal + 2. letter case conversion + 3. oov tagger + ''' + + def __init__( + self, + remove_puncts: bool = False, + to_upper: bool = False, + to_lower: bool = False, + tag_oov: bool = False, + ): + super().__init__(name="PostProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if remove_puncts: + remove_puncts_graph = pynutil.delete( + pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) + ) + graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() + + if to_upper or to_lower: + if to_upper: + conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) + else: + conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) + + graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() + + if tag_oov: + zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) + zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) + + zh_charset = ( + zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) + ) + en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE + charset = zh_charset | en_charset + + with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: + tags = f.readline().strip().split('\t') + assert len(tags) == 2 + ltag, rtag = tags + + oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) + tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) + graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/time.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa26500c5c3af3c01c50abe1ea5c85e24c8d396 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/time.py @@ -0,0 +1,93 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time e.g. + tokens { time { hour: "五点" } } -> 五点 + tokens { time { minute: "三分" }' } -> 三分 + tokens { time { hour: "五点" minute: "三分" } } -> 五点三分 + tokens { time { affix: "am" hour: "五点" verb: "差" minute: "三分" }' } -> 早上五点差三分 + tokens { time { affix: "am" hour: "一点" minute: "三分" } } -> 深夜一点三分 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + # data imported to process am/pm into mandarin + alphabet_am = pynini.string_file(get_abs_path("data/time/AM.tsv")) + alphabet_pm = pynini.string_file(get_abs_path("data/time/PM.tsv")) + + # fundamental components + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_component = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + second_component = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph_regular = ( + hour_component + | minute_component + | second_component + | (hour_component + delete_space + minute_component + delete_space + second_component) + | (hour_component + delete_space + minute_component) + | (hour_component + delete_space + second_component) + | (minute_component + delete_space + second_component) + ) + + # back count 三点差五分 + delete_verb = pynutil.delete("morphosyntactic_features: \"") + pynini.accep("差") + pynutil.delete("\"") + graph_back_count = ( + ( + pynini.closure(delete_verb + pynutil.insert(' ')) + + hour_component + + delete_space + + pynini.closure(delete_verb) + + delete_space + + minute_component + ) + | ( + pynini.closure(delete_verb + pynutil.insert(' ')) + + hour_component + + delete_space + + pynini.closure(delete_verb) + + delete_space + + second_component + ) + | ( + pynini.closure(delete_verb + pynutil.insert(' ')) + + hour_component + + delete_space + + pynini.closure(delete_verb) + + delete_space + + minute_component + + delete_space + + second_component + ) + ) + + graph = graph_regular | graph_back_count + + delete_suffix = pynutil.delete("suffix: \"") + pynini.closure(alphabet_am | alphabet_pm) + pynutil.delete("\"") + graph = graph | (graph + delete_space + delete_suffix) + + final_graph = graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..166f8f4c240767209f6411f7e8ad34528473c5f2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py @@ -0,0 +1,69 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, delete_space +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.cardinal import CardinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.date import DateFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.decimal import DecimalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.fraction import FractionFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.measure import MeasureFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.money import MoneyFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.ordinal import OrdinalFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.time import TimeFst +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.whitelist import Whitelist +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + date = DateFst(deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + fraction = FractionFst(decimal=decimal, deterministic=deterministic) + money = MoneyFst(decimal=decimal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + whitelist = Whitelist(deterministic=deterministic) + + graph = pynini.union( + date.fst, + cardinal.fst, + ordinal.fst, + decimal.fst, + fraction.fst, + word.fst, + money.fst, + measure.fst, + time.fst, + whitelist.fst, + ) + graph = pynini.closure(delete_space) + graph + pynini.closure(delete_space) + + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..99cd17e7cdea96d2e4ee765f106c345a0b9ba0a2 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, delete_space, generator_main +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.postprocessor import PostProcessor +from zonos2.vendor.nemo_text_processing.text_normalization.zh.verbalizers.verbalize import VerbalizeFst + +# from nemo.utils import logging + + +class VerbalizeFinalFst(GraphFst): + """ """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) + + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) + + self.fst = (verbalizer @ postprocessor.fst).optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..4086e90f0b4a827af292ef8d22ff9b96d419c12b --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class Whitelist(GraphFst): + ''' + tokens { whitelist: "ATM" } -> A T M + ''' + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + remove_erhua = pynutil.delete("erhua: \"") + pynutil.delete("儿") + pynutil.delete("\"") + whitelist = pynutil.delete("name: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph = remove_erhua | whitelist + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/word.py b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..34aebd8c596f435a732a79f220d7909fa01e9836 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/text_normalization/zh/verbalizers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from zonos2.vendor.nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + ''' + tokens { char: "你" } -> 你 + ''' + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="char", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") + self.fst = graph.optimize() diff --git a/zonos2/vendor/nemo_text_processing/utils/__init__.py b/zonos2/vendor/nemo_text_processing/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94bb553554342685c689a79b62903b28fa7a5bee --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATE. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos2.vendor.nemo_text_processing.utils.logging import logger diff --git a/zonos2/vendor/nemo_text_processing/utils/logging.py b/zonos2/vendor/nemo_text_processing/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..96187bfdf81804e5ab86ddd3f64fb3bb7e6e37e6 --- /dev/null +++ b/zonos2/vendor/nemo_text_processing/utils/logging.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +# overriding with the library specific logger, so that it's possible to +# customize in any downstream applications +logger = logging.getLogger("NeMo-text-processing") +c_handler = logging.StreamHandler() +logger.addHandler(c_handler) +logger.setLevel(logging.INFO) +logFormatter = logging.Formatter(fmt=' %(name)s :: %(levelname)-8s :: %(message)s') +c_handler.setFormatter(logFormatter)