Mark-Engineer-V2-8B / tokenizer_config.json
miceworks's picture
feat: upload TOKENIZER_CONFIG weights (tokenizer_config.json)
1904561 verified
Raw History Blame Contribute Delete
1.02 kB
{
"vocab_size": 4311,
"bos_token_id": 1,
"eos_token_id": 2,
"special_tokens": {
"<|pad|>": 0,
"<|bos|>": 1,
"<|eos|>": 2,
"<think>": 3,
"</think>": 4,
"[RETRIEVE]": 5,
"[/RETRIEVE]": 6,
"[EVALUATE]": 7,
"[ACCEPT]": 8,
"[REJECT]": 9,
"<|vision_start|>": 10,
"<|vision_patch|>": 11,
"<|vision_end|>": 12,
"<|voice_stream|>": 13,
"[UNKNOWN]": 14,
"[WEB_SEARCH]": 15,
"[/WEB_SEARCH]": 16,
"<|rag_context|>": 17,
"<|/rag_context|>": 18,
"<|mcp_call|>": 19,
"<|mcp_response|>": 20,
"<|citation|>": 21,
"<|/citation|>": 22
},
"supported_languages": [
"Rust",
"Python",
"Ruby",
"JavaScript",
"TypeScript",
"CSS",
"PHP",
"Kotlin",
"Java",
"Go",
"C++",
"C",
"C#",
"Assembly (x86_64/ARM64)",
"SQL",
"Shell"
],
"natural_languages": [
"pt",
"en",
"es",
"fr",
"de",
"it",
"ru",
"ja",
"ko",
"ar",
"hi",
"zh-Hans"
]
}