Upload tokenizer.py with huggingface_hub

tokenizer.py

@@ -13,7 +13,9 @@ from tokenizers.decoders import ByteLevel as ByteLevelDecoder
 from tokenizers import AddedToken
 
 SPECIAL = ["<pad>", "<bos>", "<eos>", "<mask>",
-            "<think>", "</think>", "<tool>", "</tool>", "<result>"]
+            "<think>", "</think>", "<tool>", "</tool>", "<result>",
+            # learned secondary-memory tokens (side store, not in weights)
+            "<mem_write>", "<mem_read>", "<mem_kv>", "<mem_evict>"]
 
 
 class YKTokenizer:
@@ -84,7 +86,15 @@ class YKTokenizer:
     @property
     def endtool_id(self): return self._ids["</tool>"]
     @property
-    def result_id(self): return self._ids["<result>"]
+    def result_id(self):  return self._ids["<result>"]
+    @property
+    def mem_write_id(self):  return self._ids["<mem_write>"]
+    @property
+    def mem_read_id(self):   return self._ids["<mem_read>"]
+    @property
+    def mem_kv_id(self):     return self._ids["<mem_kv>"]
+    @property
+    def mem_evict_id(self):  return self._ids["<mem_evict>"]
 
     # ---- encode / decode ------------------------------------------------
     def encode(self, text, add_special=False):