pluto-2.1 / tokenizer /tokenizer_config.json
henryphilf's picture
Release Pluto-2.1 1.7B: standalone decision model
5b31db0 verified
Raw History Blame Contribute Delete
6.83 kB
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|decision_000|>",
"<|decision_001|>",
"<|decision_002|>",
"<|decision_003|>",
"<|decision_004|>",
"<|decision_005|>",
"<|decision_006|>",
"<|decision_007|>",
"<|decision_008|>",
"<|decision_009|>",
"<|decision_010|>",
"<|decision_011|>",
"<|decision_012|>",
"<|decision_013|>",
"<|decision_014|>",
"<|decision_015|>",
"<|decision_016|>",
"<|decision_017|>",
"<|decision_018|>",
"<|decision_019|>",
"<|decision_020|>",
"<|decision_021|>",
"<|decision_022|>",
"<|decision_023|>",
"<|decision_024|>",
"<|decision_025|>",
"<|decision_026|>",
"<|decision_027|>",
"<|decision_028|>",
"<|decision_029|>",
"<|decision_030|>",
"<|decision_031|>",
"<|decision_032|>",
"<|decision_033|>",
"<|decision_034|>",
"<|decision_035|>",
"<|decision_036|>",
"<|decision_037|>",
"<|decision_038|>",
"<|decision_039|>",
"<|decision_040|>",
"<|decision_041|>",
"<|decision_042|>",
"<|decision_043|>",
"<|decision_044|>",
"<|decision_045|>",
"<|decision_046|>",
"<|decision_047|>",
"<|decision_048|>",
"<|decision_049|>",
"<|decision_050|>",
"<|decision_051|>",
"<|decision_052|>",
"<|decision_053|>",
"<|decision_054|>",
"<|decision_055|>",
"<|decision_056|>",
"<|decision_057|>",
"<|decision_058|>",
"<|decision_059|>",
"<|decision_060|>",
"<|decision_061|>",
"<|decision_062|>",
"<|decision_063|>",
"<|decision_064|>",
"<|decision_065|>",
"<|decision_066|>",
"<|decision_067|>",
"<|decision_068|>",
"<|decision_069|>",
"<|decision_070|>",
"<|decision_071|>",
"<|decision_072|>",
"<|decision_073|>",
"<|decision_074|>",
"<|decision_075|>",
"<|decision_076|>",
"<|decision_077|>",
"<|decision_078|>",
"<|decision_079|>",
"<|decision_080|>",
"<|decision_081|>",
"<|decision_082|>",
"<|decision_083|>",
"<|decision_084|>",
"<|decision_085|>",
"<|decision_086|>",
"<|decision_087|>",
"<|decision_088|>",
"<|decision_089|>",
"<|decision_090|>",
"<|decision_091|>",
"<|decision_092|>",
"<|decision_093|>",
"<|decision_094|>",
"<|decision_095|>",
"<|decision_096|>",
"<|decision_097|>",
"<|decision_098|>",
"<|decision_099|>",
"<|decision_100|>",
"<|decision_101|>",
"<|decision_102|>",
"<|decision_103|>",
"<|decision_104|>",
"<|decision_105|>",
"<|decision_106|>",
"<|decision_107|>",
"<|decision_108|>",
"<|decision_109|>",
"<|decision_110|>",
"<|decision_111|>",
"<|decision_112|>",
"<|decision_113|>",
"<|decision_114|>",
"<|decision_115|>",
"<|decision_116|>",
"<|decision_117|>",
"<|decision_118|>",
"<|decision_119|>",
"<|decision_120|>",
"<|decision_121|>",
"<|decision_122|>",
"<|decision_123|>",
"<|decision_124|>",
"<|decision_125|>",
"<|decision_126|>",
"<|decision_127|>",
"<|decision_128|>",
"<|decision_129|>",
"<|decision_130|>",
"<|decision_131|>",
"<|decision_132|>",
"<|decision_133|>",
"<|decision_134|>",
"<|decision_135|>",
"<|decision_136|>",
"<|decision_137|>",
"<|decision_138|>",
"<|decision_139|>",
"<|decision_140|>",
"<|decision_141|>",
"<|decision_142|>",
"<|decision_143|>",
"<|decision_144|>",
"<|decision_145|>",
"<|decision_146|>",
"<|decision_147|>",
"<|decision_148|>",
"<|decision_149|>",
"<|decision_150|>",
"<|decision_151|>",
"<|decision_152|>",
"<|decision_153|>",
"<|decision_154|>",
"<|decision_155|>",
"<|decision_156|>",
"<|decision_157|>",
"<|decision_158|>",
"<|decision_159|>",
"<|decision_160|>",
"<|decision_161|>",
"<|decision_162|>",
"<|decision_163|>",
"<|decision_164|>",
"<|decision_165|>",
"<|decision_166|>",
"<|decision_167|>",
"<|decision_168|>",
"<|decision_169|>",
"<|decision_170|>",
"<|decision_171|>",
"<|decision_172|>",
"<|decision_173|>",
"<|decision_174|>",
"<|decision_175|>",
"<|decision_176|>",
"<|decision_177|>",
"<|decision_178|>",
"<|decision_179|>",
"<|decision_180|>",
"<|decision_181|>",
"<|decision_182|>",
"<|decision_183|>",
"<|decision_184|>",
"<|decision_185|>",
"<|decision_186|>",
"<|decision_187|>",
"<|decision_188|>",
"<|decision_189|>",
"<|decision_190|>",
"<|decision_191|>",
"<|decision_192|>",
"<|decision_193|>",
"<|decision_194|>",
"<|decision_195|>",
"<|decision_196|>",
"<|decision_197|>",
"<|decision_198|>",
"<|decision_199|>",
"<|decision_200|>",
"<|decision_201|>",
"<|decision_202|>",
"<|decision_203|>",
"<|decision_204|>",
"<|decision_205|>",
"<|decision_206|>",
"<|decision_207|>",
"<|decision_208|>",
"<|decision_209|>",
"<|decision_210|>",
"<|decision_211|>",
"<|decision_212|>",
"<|decision_213|>",
"<|decision_214|>",
"<|decision_215|>",
"<|decision_216|>",
"<|decision_217|>",
"<|decision_218|>",
"<|decision_219|>",
"<|decision_220|>",
"<|decision_221|>",
"<|decision_222|>",
"<|decision_223|>",
"<|decision_224|>",
"<|decision_225|>",
"<|decision_226|>",
"<|decision_227|>",
"<|decision_228|>",
"<|decision_229|>",
"<|decision_230|>",
"<|decision_231|>",
"<|decision_232|>",
"<|decision_233|>",
"<|decision_234|>",
"<|decision_235|>",
"<|decision_236|>",
"<|decision_237|>",
"<|decision_238|>",
"<|decision_239|>",
"<|decision_240|>",
"<|decision_241|>",
"<|decision_242|>",
"<|decision_243|>",
"<|decision_244|>",
"<|decision_245|>",
"<|decision_246|>",
"<|decision_247|>",
"<|decision_248|>",
"<|decision_249|>",
"<|decision_250|>",
"<|decision_251|>",
"<|decision_252|>",
"<|decision_253|>",
"<|decision_254|>",
"<|decision_255|>"
],
"is_local": true,
"local_files_only": false,
"model_max_length": 131072,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}