mlboydaisuke commited on
Commit
dd4de01
·
verified ·
1 Parent(s): dcd46e4

tokenizer_config: inline the gemma-4 chat template (ships as separate chat_template.jinja upstream, which swift-transformers does not read; without it prompts raw-encode and generation degenerates to turn markers)

Browse files
gpu-pipelined-b2/gemma4_e2b_qat_decode_int4lin_tbl_aotc_h18p/tokenizer/tokenizer_config.json CHANGED
@@ -92,5 +92,6 @@
92
  "str_token": "<|tool_response>",
93
  "think_token": "<|think|>",
94
  "tokenizer_class": "GemmaTokenizer",
95
- "unk_token": "<unk>"
96
- }
 
 
92
  "str_token": "<|tool_response>",
93
  "think_token": "<|think|>",
94
  "tokenizer_class": "GemmaTokenizer",
95
+ "unk_token": "<unk>",
96
+ "chat_template": "{{ bos_token }}{% for message in messages %}<|turn>{% if message['role'] == 'assistant' %}model{% else %}{{ message['role'] }}{% endif %}\n{{ message['content'] }}<turn|>\n{% endfor %}{% if add_generation_prompt %}<|turn>model\n{% endif %}"
97
+ }