Spell the reasoning tokens <think> / </think> (non-special, same ids 32753 / 32754)

encoding.json

@@ -1,32 +1,32 @@
-{
-  "name": "luau-coder",
-  "pat_str": "(?<=\\p{L})['\\u2019](?i:s|t|re|ve|m|ll|d)(?![\\p{L}\\p{N}])|[ ]?[\\p{L}\\p{M}_][\\p{L}\\p{M}\\p{N}_]*| ?\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}_]+|[ \\t]*[\\r\\n]+[ \\t]*|\\s+(?!\\S)|\\s+",
-  "tiktoken_file": "tiktoken.model",
-  "special_tokens": {
-    "<|endoftext|>": 32744,
-    "<|pad|>": 32745,
-    "<|bos|>": 32746,
-    "<|fim_prefix|>": 32747,
-    "<|fim_middle|>": 32748,
-    "<|fim_suffix|>": 32749,
-    "<|fim_pad|>": 32750,
-    "<|im_start|>": 32751,
-    "<|im_end|>": 32752,
-    "<|think|>": 32753,
-    "<|/think|>": 32754,
-    "<|tool_call|>": 32755,
-    "<|/tool_call|>": 32756,
-    "<|tool_response|>": 32757,
-    "<|/tool_response|>": 32758,
-    "<|repo_name|>": 32759,
-    "<|file_sep|>": 32760,
-    "<|file_path|>": 32761,
-    "<|code_start|>": 32762,
-    "<|code_end|>": 32763,
-    "<|reserved_20|>": 32764,
-    "<|reserved_21|>": 32765,
-    "<|reserved_22|>": 32766,
-    "<|reserved_23|>": 32767
-  },
-  "vocab_size": 32768
-}
+{
+  "name": "luau-coder",
+  "pat_str": "(?<=\\p{L})['\\u2019](?i:s|t|re|ve|m|ll|d)(?![\\p{L}\\p{N}])|[ ]?[\\p{L}\\p{M}_][\\p{L}\\p{M}\\p{N}_]*| ?\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}_]+|[ \\t]*[\\r\\n]+[ \\t]*|\\s+(?!\\S)|\\s+",
+  "tiktoken_file": "tiktoken.model",
+  "special_tokens": {
+    "<|endoftext|>": 32744,
+    "<|pad|>": 32745,
+    "<|bos|>": 32746,
+    "<|fim_prefix|>": 32747,
+    "<|fim_middle|>": 32748,
+    "<|fim_suffix|>": 32749,
+    "<|fim_pad|>": 32750,
+    "<|im_start|>": 32751,
+    "<|im_end|>": 32752,
+    "<think>": 32753,
+    "</think>": 32754,
+    "<|tool_call|>": 32755,
+    "<|/tool_call|>": 32756,
+    "<|tool_response|>": 32757,
+    "<|/tool_response|>": 32758,
+    "<|repo_name|>": 32759,
+    "<|file_sep|>": 32760,
+    "<|file_path|>": 32761,
+    "<|code_start|>": 32762,
+    "<|code_end|>": 32763,
+    "<|reserved_20|>": 32764,
+    "<|reserved_21|>": 32765,
+    "<|reserved_22|>": 32766,
+    "<|reserved_23|>": 32767
+  },
+  "vocab_size": 32768
+}

special_tokens_map.json

@@ -1,24 +1,22 @@
-{
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|pad|>",
-  "additional_special_tokens": [
-    "<|bos|>",
-    "<|fim_prefix|>",
-    "<|fim_middle|>",
-    "<|fim_suffix|>",
-    "<|fim_pad|>",
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|think|>",
-    "<|/think|>",
-    "<|tool_call|>",
-    "<|/tool_call|>",
-    "<|tool_response|>",
-    "<|/tool_response|>",
-    "<|repo_name|>",
-    "<|file_sep|>",
-    "<|file_path|>",
-    "<|code_start|>",
-    "<|code_end|>"
-  ]
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "additional_special_tokens": [
+    "<|bos|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|/tool_call|>",
+    "<|tool_response|>",
+    "<|/tool_response|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|file_path|>",
+    "<|code_start|>",
+    "<|code_end|>"
+  ]
 }

specials.json

@@ -1,26 +1,26 @@
-{
-  "<|endoftext|>": 32744,
-  "<|pad|>": 32745,
-  "<|bos|>": 32746,
-  "<|fim_prefix|>": 32747,
-  "<|fim_middle|>": 32748,
-  "<|fim_suffix|>": 32749,
-  "<|fim_pad|>": 32750,
-  "<|im_start|>": 32751,
-  "<|im_end|>": 32752,
-  "<|think|>": 32753,
-  "<|/think|>": 32754,
-  "<|tool_call|>": 32755,
-  "<|/tool_call|>": 32756,
-  "<|tool_response|>": 32757,
-  "<|/tool_response|>": 32758,
-  "<|repo_name|>": 32759,
-  "<|file_sep|>": 32760,
-  "<|file_path|>": 32761,
-  "<|code_start|>": 32762,
-  "<|code_end|>": 32763,
-  "<|reserved_20|>": 32764,
-  "<|reserved_21|>": 32765,
-  "<|reserved_22|>": 32766,
-  "<|reserved_23|>": 32767
+{
+  "<|endoftext|>": 32744,
+  "<|pad|>": 32745,
+  "<|bos|>": 32746,
+  "<|fim_prefix|>": 32747,
+  "<|fim_middle|>": 32748,
+  "<|fim_suffix|>": 32749,
+  "<|fim_pad|>": 32750,
+  "<|im_start|>": 32751,
+  "<|im_end|>": 32752,
+  "<think>": 32753,
+  "</think>": 32754,
+  "<|tool_call|>": 32755,
+  "<|/tool_call|>": 32756,
+  "<|tool_response|>": 32757,
+  "<|/tool_response|>": 32758,
+  "<|repo_name|>": 32759,
+  "<|file_sep|>": 32760,
+  "<|file_path|>": 32761,
+  "<|code_start|>": 32762,
+  "<|code_end|>": 32763,
+  "<|reserved_20|>": 32764,
+  "<|reserved_21|>": 32765,
+  "<|reserved_22|>": 32766,
+  "<|reserved_23|>": 32767
 }

tokenizer_config.json

@@ -1,32 +1,30 @@
-{
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "model_max_length": 1000000000,
-  "clean_up_tokenization_spaces": false,
-  "add_prefix_space": false,
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "bos_token": null,
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|pad|>",
-  "unk_token": null,
-  "additional_special_tokens": [
-    "<|bos|>",
-    "<|fim_prefix|>",
-    "<|fim_middle|>",
-    "<|fim_suffix|>",
-    "<|fim_pad|>",
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|think|>",
-    "<|/think|>",
-    "<|tool_call|>",
-    "<|/tool_call|>",
-    "<|tool_response|>",
-    "<|/tool_response|>",
-    "<|repo_name|>",
-    "<|file_sep|>",
-    "<|file_path|>",
-    "<|code_start|>",
-    "<|code_end|>"
-  ]
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "model_max_length": 1000000000,
+  "clean_up_tokenization_spaces": false,
+  "add_prefix_space": false,
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "bos_token": null,
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "unk_token": null,
+  "additional_special_tokens": [
+    "<|bos|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|/tool_call|>",
+    "<|tool_response|>",
+    "<|/tool_response|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|file_path|>",
+    "<|code_start|>",
+    "<|code_end|>"
+  ]
 }