Create tokenizer.py

tokenizer.py

@@ -0,0 +1,11 @@
+import torch
+from transformers import AutoTokenizer
+
+class BathSalt1DaedalusPhi3Tokenizer(AutoTokenizer):
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+    def encode(self, text, return_tensors="pt"):
+        encoding = super().encode(text, return_tensors=return_tensors)
+        return encoding