[FEAT] Add CBTTokenizer class alias
Browse files- tokenizer.py +5 -0
tokenizer.py
CHANGED
|
@@ -107,3 +107,8 @@ class ScaledTokenizer:
|
|
| 107 |
else:
|
| 108 |
chars.append("<UNK>")
|
| 109 |
return "".join(chars)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
else:
|
| 108 |
chars.append("<UNK>")
|
| 109 |
return "".join(chars)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# Aliases for Character Based Transformer (CBT)
|
| 113 |
+
CBTTokenizer = ScaledTokenizer
|
| 114 |
+
|