File size: 1,581 Bytes
a14a820 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | {
"vocab_size": 32032,
"bos_token_id": 1,
"eos_token_id": 2,
"special_tokens": {
"<|pad|>": 0,
"<|bos|>": 1,
"<|eos|>": 2,
"<think>": 3,
"</think>": 4,
"[RETRIEVE]": 5,
"[/RETRIEVE]": 6,
"[EVALUATE]": 7,
"[ACCEPT]": 8,
"[REJECT]": 9,
"<|vision_start|>": 10,
"<|vision_patch|>": 11,
"<|vision_end|>": 12,
"<|voice_stream|>": 13,
"[UNKNOWN]": 14,
"[WEB_SEARCH]": 15,
"[/WEB_SEARCH]": 16,
"<|rag_context|>": 17,
"<|/rag_context|>": 18,
"<|mcp_call|>": 19,
"<|mcp_response|>": 20,
"<|citation|>": 21,
"<|/citation|>": 22
},
"supported_languages": [
"Rust",
"Python",
"Ruby",
"JavaScript",
"TypeScript",
"CSS",
"PHP",
"Kotlin",
"Java",
"Go",
"C++",
"C",
"C#",
"Assembly (x86_64/ARM64)",
"SQL",
"Shell"
],
"natural_languages": [
"pt",
"en",
"es",
"fr",
"de",
"it",
"ru",
"ja",
"ko",
"ar",
"hi",
"zh-Hans"
],
"bos_token": "<|bos|>",
"eos_token": "<|eos|>",
"pad_token": "<|pad|>",
"pad_token_id": 0,
"model_max_length": 8192,
"additional_special_tokens": [
"<think>",
"</think>",
"[RETRIEVE]",
"[/RETRIEVE]",
"[EVALUATE]",
"[ACCEPT]",
"[REJECT]",
"<|vision_start|>",
"<|vision_patch|>",
"<|vision_end|>",
"<|voice_stream|>",
"<|rag_context|>",
"<|/rag_context|>",
"<|mcp_call|>",
"<|mcp_response|>",
"<|citation|>",
"<|/citation|>"
],
"tokenizer_class": "PreTrainedTokenizerFast"
}
|