Sovereign Tokenizer Upgrade: H100 Tensor-Core Aligned 40,960 BPE (special_tokens_map.json)

special_tokens_map.json

@@ -0,0 +1,64 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "additional_special_tokens": [
+    "<image>",
+    "</image>",
+    "<audio>",
+    "</audio>",
+    "<video>",
+    "</video>",
+    "<click>",
+    "</click>",
+    "<touch>",
+    "</touch>",
+    "<box3d>",
+    "</box3d>",
+    "<frame>",
+    "</frame>",
+    "<document>",
+    "</document>",
+    "<page>",
+    "</page>",
+    "<table>",
+    "</table>",
+    "<sheet>",
+    "</sheet>",
+    "<chart>",
+    "</chart>",
+    "<layout>",
+    "</layout>",
+    "<ocr>",
+    "</ocr>",
+    "<system>",
+    "</system>",
+    "<user>",
+    "</user>",
+    "<assistant>",
+    "</assistant>",
+    "<|sovereign_reserved_0|>",
+    "<|sovereign_reserved_1|>",
+    "<|sovereign_reserved_2|>",
+    "<|sovereign_reserved_3|>",
+    "<|sovereign_reserved_4|>",
+    "<|sovereign_reserved_5|>",
+    "<|sovereign_reserved_6|>",
+    "<|sovereign_reserved_7|>",
+    "<|sovereign_reserved_8|>",
+    "<|sovereign_reserved_9|>",
+    "<|sovereign_reserved_10|>",
+    "<|sovereign_reserved_11|>",
+    "<|sovereign_reserved_12|>",
+    "<|sovereign_reserved_13|>",
+    "<|sovereign_reserved_14|>",
+    "<|sovereign_reserved_15|>",
+    "<|sovereign_reserved_16|>",
+    "<|sovereign_reserved_17|>",
+    "<|sovereign_reserved_18|>",
+    "<|sovereign_reserved_19|>",
+    "<|sovereign_reserved_20|>",
+    "<|sovereign_reserved_21|>"
+  ]
+}