TwIL-LM3-Pro / tokenizer_config.json
anurag051194's picture
Meridian-smaller: Granite 4.2 MGPO step 2580 and GGUF builds
b95b830 verified
Raw History Blame Contribute Delete
27 kB
{
"add_prefix_space": false,
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"local_files_only": false,
"model_max_length": 131072,
"pad_token": "<|im_end|>",
"padding_side": "left",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<unk>",
"added_tokens_decoder": {
"100256": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100257": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100258": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100259": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100260": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100261": {
"content": "<|fim_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100262": {
"content": "<|filename|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100263": {
"content": "<|reponame|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100264": {
"content": "<|unused_11|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100265": {
"content": "<|unused_12|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100266": {
"content": "<|unused_13|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100267": {
"content": "<|unused_14|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100268": {
"content": "<|unused_15|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100269": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100270": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100271": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100272": {
"content": "<tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100273": {
"content": "</tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100274": {
"content": "<think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100275": {
"content": "</think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"100276": {
"content": "[INST]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100277": {
"content": "[/INST]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100278": {
"content": "[AVAILABLE_TOOLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100279": {
"content": "[/AVAILABLE_TOOLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100280": {
"content": "[TOOL_RESULTS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100281": {
"content": "[/TOOL_RESULTS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100282": {
"content": "[TOOL_CALLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100283": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100284": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100285": {
"content": "<|unused_16|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100286": {
"content": "<|unused_17|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100287": {
"content": "<|unused_18|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100288": {
"content": "<|unused_19|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100289": {
"content": "<|unused_20|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100290": {
"content": "<|unused_21|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100291": {
"content": "<|unused_22|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100292": {
"content": "<|unused_23|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100293": {
"content": "<|unused_24|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100294": {
"content": "<|unused_25|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100295": {
"content": "<|unused_26|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100296": {
"content": "<|unused_27|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100297": {
"content": "<|unused_28|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100298": {
"content": "<|unused_29|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100299": {
"content": "<|unused_30|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100300": {
"content": "<|unused_31|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100301": {
"content": "<|unused_32|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100302": {
"content": "<|unused_33|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100303": {
"content": "<|unused_34|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100304": {
"content": "<|unused_35|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100305": {
"content": "<|unused_36|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100306": {
"content": "<|unused_37|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100307": {
"content": "<|unused_38|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100308": {
"content": "<|unused_39|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100309": {
"content": "<|unused_40|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100310": {
"content": "<|unused_41|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100311": {
"content": "<|unused_42|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100312": {
"content": "<|unused_43|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100313": {
"content": "<|unused_44|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100314": {
"content": "<|unused_45|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100315": {
"content": "<|unused_46|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100316": {
"content": "<|unused_47|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100317": {
"content": "<|unused_48|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100318": {
"content": "<|unused_49|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100319": {
"content": "<|unused_50|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100320": {
"content": "<|unused_51|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100321": {
"content": "<|unused_52|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100322": {
"content": "<|unused_53|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100323": {
"content": "<|unused_54|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100324": {
"content": "<|unused_55|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100325": {
"content": "<|unused_56|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100326": {
"content": "<|unused_57|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100327": {
"content": "<|unused_58|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100328": {
"content": "<|unused_59|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100329": {
"content": "<|unused_60|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100330": {
"content": "<|unused_61|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100331": {
"content": "<|unused_62|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100332": {
"content": "<|unused_63|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100333": {
"content": "<|unused_64|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100334": {
"content": "<|unused_65|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100335": {
"content": "<|unused_66|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100336": {
"content": "<|unused_67|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100337": {
"content": "<|unused_68|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100338": {
"content": "<|unused_69|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100339": {
"content": "<|unused_70|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100340": {
"content": "<|unused_71|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100341": {
"content": "<|unused_72|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100342": {
"content": "<|unused_73|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100343": {
"content": "<|unused_74|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100344": {
"content": "<|unused_75|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100345": {
"content": "<|unused_76|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100346": {
"content": "<|unused_77|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100347": {
"content": "<|unused_78|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100348": {
"content": "<|unused_79|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100349": {
"content": "<|unused_80|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100350": {
"content": "<|unused_81|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100351": {
"content": "<|unused_82|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"chat_template": "{%- macro tool_to_json(tool) -%}\n {%- set ns_tool = namespace(first=true) -%}\n {{- '{' -}}\n {%- for k, v in tool.items() -%}\n {%- if k != 'defer_loading' and k != 'strict' -%}\n {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}\n {%- set ns_tool.first = false -%}\n {{- '\"' ~ k ~ '\": ' ~ (v | tojson(ensure_ascii=False)) -}}\n {%- endif -%}\n {%- endfor -%}\n {{- '}' -}}\n{%- endmacro -%}\n{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}\n{%- set low_effort = low_effort if low_effort is defined else False %}\n{%- if reasoning_effort is defined and reasoning_effort is not none %}\n {%- set low_effort = reasoning_effort == \"low\" %}\n{%- endif %}\n{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \"<|im_start|>system\\n\" }}\n {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n {%- if system_message is defined and system_message | length > 0 %}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {{- \"<tools>\" }}\n {%- for tool in tools %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {%- if tool.defer_loading is not defined or not tool.defer_loading %}\n {{- '\\n' ~ tool_to_json(tool) }}\n {%- endif %}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == \"assistant\" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n {%- else %}\n {%- set content = message.content | default('', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if '<think>' not in content and '</think>' not in content -%}\n {%- set content = \"<think></think>\" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- '<|im_start|>assistant\\n' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ '\\n' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if '</think>' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split('</think>')[-1] %}\n {%- elif '<think>' in c %}\n {# If <think> was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split('<think>')[0] %}\n {%- endif %}\n {%- set c = \"<think></think>\" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ '\\n' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- \"<think></think>\" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ '\\n</parameter>\\n' -}}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>\\n' -}}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- else %}\n {# Assistant message doesn't have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }}\n {%- else %}\n {%- set c = (content | default('', true) | string) %}\n {%- if '<think>' in c and '</think>' in c %}\n {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == \"user\" or message.role == \"system\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- set content = message.content | string %}\n {%- if message.role == \"user\" and loop.index0 == ns.last_user_idx and low_effort %}\n {{- content + '\\n\\n{reasoning effort: low}' }}\n {%- else %}\n {{- content }}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user\\n' }}\n {%- endif %}\n {{- '<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think></think>' }}\n {%- endif %}\n{%- endif %}\n"
}