File size: 1,025 Bytes
a4c3ef3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | {
"name": "DOAIxILPRL-KU/nep-tokenizer",
"tier": "64k",
"vocab_size": 64053,
"version": "v2.0",
"institution": "Department of Artificial Intelligence (DOAI) & Information and Language Processing Research Lab (ILPRL), Kathmandu University",
"description": "Standard 64k vocabulary for Nepali & Multilingual Large Language Models",
"model_type": "byte-level BPE",
"backend": "tokenizers (Rust PreTrainedTokenizerFast)",
"normalization": "NFC",
"pre_tokenizer": "Isolated Digit Grouping [0-9०-९]{1,3} + Mark-Aware Unicode Regex [\\p{L}\\p{M}]+ + ByteLevel",
"decoder": "ByteLevel",
"devanagari_block_coverage": "100.0% (all 128 assigned codepoints guaranteed)",
"matra_cohesion_rate": "100.0%",
"roundtrip_fidelity": "100.0%",
"special_tokens": [
"<|pad|>",
"<|bos|>",
"<|eos|>",
"<|unk|>",
"<|sep|>",
"<|start_turn|>",
"<|end_turn|>",
"<|think|>",
"<|/think|>",
"<|tool_call|>",
"<|/tool_call|>",
"<|tool_result|>",
"<|/tool_result|>"
]
}
|