| { |
| "add_prefix_space": false, |
| "backend": "tokenizers", |
| "bos_token": "<|im_start|>", |
| "clean_up_tokenization_spaces": false, |
| "end_of_utterance_token": "<end_of_utterance>", |
| "eos_token": "<end_of_utterance>", |
| "errors": "replace", |
| "fake_image_token": "<fake_token_around_image>", |
| "global_image_token": "<global-img>", |
| "image_token": "<image>", |
| "is_local": true, |
| "legacy": false, |
| "max_length": 48, |
| "model_max_length": 8192, |
| "model_specific_special_tokens": { |
| "end_of_utterance_token": "<end_of_utterance>", |
| "fake_image_token": "<fake_token_around_image>", |
| "global_image_token": "<global-img>", |
| "image_token": "<image>" |
| }, |
| "pad_to_multiple_of": null, |
| "pad_token": "<|im_end|>", |
| "pad_token_type_id": 0, |
| "padding_side": "right", |
| "processor_class": "SmolVLMProcessor", |
| "stride": 0, |
| "tokenizer_class": "GPT2Tokenizer", |
| "truncation_side": "left", |
| "truncation_strategy": "longest_first", |
| "unk_token": "<|endoftext|>", |
| "vocab_size": 49152 |
| } |
|
|
