Release Nep-Tokenizer v2 (64k): Native Hugging Face BPE with Mark-Aware Pre-tokenization and 100% Devanagari Block Coverage
a4c3ef3 | { | |
| "name": "DOAIxILPRL-KU/nep-tokenizer", | |
| "tier": "64k", | |
| "vocab_size": 64053, | |
| "version": "v2.0", | |
| "institution": "Department of Artificial Intelligence (DOAI) & Information and Language Processing Research Lab (ILPRL), Kathmandu University", | |
| "description": "Standard 64k vocabulary for Nepali & Multilingual Large Language Models", | |
| "model_type": "byte-level BPE", | |
| "backend": "tokenizers (Rust PreTrainedTokenizerFast)", | |
| "normalization": "NFC", | |
| "pre_tokenizer": "Isolated Digit Grouping [0-9०-९]{1,3} + Mark-Aware Unicode Regex [\\p{L}\\p{M}]+ + ByteLevel", | |
| "decoder": "ByteLevel", | |
| "devanagari_block_coverage": "100.0% (all 128 assigned codepoints guaranteed)", | |
| "matra_cohesion_rate": "100.0%", | |
| "roundtrip_fidelity": "100.0%", | |
| "special_tokens": [ | |
| "<|pad|>", | |
| "<|bos|>", | |
| "<|eos|>", | |
| "<|unk|>", | |
| "<|sep|>", | |
| "<|start_turn|>", | |
| "<|end_turn|>", | |
| "<|think|>", | |
| "<|/think|>", | |
| "<|tool_call|>", | |
| "<|/tool_call|>", | |
| "<|tool_result|>", | |
| "<|/tool_result|>" | |
| ] | |
| } | |