{
  "name": "DOAIxILPRL-KU/nep-tokenizer",
  "tier": "64k",
  "vocab_size": 64053,
  "version": "v2.0",
  "institution": "Department of Artificial Intelligence (DOAI) & Information and Language Processing Research Lab (ILPRL), Kathmandu University",
  "description": "Standard 64k vocabulary for Nepali & Multilingual Large Language Models",
  "model_type": "byte-level BPE",
  "backend": "tokenizers (Rust PreTrainedTokenizerFast)",
  "normalization": "NFC",
  "pre_tokenizer": "Isolated Digit Grouping [0-9०-९]{1,3} + Mark-Aware Unicode Regex [\\p{L}\\p{M}]+ + ByteLevel",
  "decoder": "ByteLevel",
  "devanagari_block_coverage": "100.0% (all 128 assigned codepoints guaranteed)",
  "matra_cohesion_rate": "100.0%",
  "roundtrip_fidelity": "100.0%",
  "special_tokens": [
    "<|pad|>",
    "<|bos|>",
    "<|eos|>",
    "<|unk|>",
    "<|sep|>",
    "<|start_turn|>",
    "<|end_turn|>",
    "<|think|>",
    "<|/think|>",
    "<|tool_call|>",
    "<|/tool_call|>",
    "<|tool_result|>",
    "<|/tool_result|>"
  ]
}