tokenizer_config: inline the gemma-4 chat template (ships as separate chat_template.jinja upstream, which swift-transformers does not read; without it prompts raw-encode and generation degenerates to turn markers)
Browse files
gpu-pipelined-b2/gemma4_e2b_qat_decode_int4lin_tbl_aotc_h18p/tokenizer/tokenizer_config.json
CHANGED
|
@@ -92,5 +92,6 @@
|
|
| 92 |
"str_token": "<|tool_response>",
|
| 93 |
"think_token": "<|think|>",
|
| 94 |
"tokenizer_class": "GemmaTokenizer",
|
| 95 |
-
"unk_token": "<unk>"
|
| 96 |
-
}
|
|
|
|
|
|
| 92 |
"str_token": "<|tool_response>",
|
| 93 |
"think_token": "<|think|>",
|
| 94 |
"tokenizer_class": "GemmaTokenizer",
|
| 95 |
+
"unk_token": "<unk>",
|
| 96 |
+
"chat_template": "{{ bos_token }}{% for message in messages %}<|turn>{% if message['role'] == 'assistant' %}model{% else %}{{ message['role'] }}{% endif %}\n{{ message['content'] }}<turn|>\n{% endfor %}{% if add_generation_prompt %}<|turn>model\n{% endif %}"
|
| 97 |
+
}
|