Add spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000

.gitattributes

@@ -74,3 +74,4 @@ spanish-english/mix/XLM-T/spanglish_MIX_tokenizer_XLMT_stages_even/tokenizer.jso
 spanish-english/mix/XLM-T/spanglish_MIX_tokenizer_XLMT_stages_evenall/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 spanish-english/mix/XLM-T/spanglish_MIX_tokenizer_XLMT_stages_neg/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 spanish-english/mix/XLM-T/spanglish_MIX_tokenizer_XLMT_stages_neu/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f4fe57cb49634fbb8ce0dde9ac48db790bcce8a3a9c28462bb4f69c0517f48
+size 17098525

spanish-english/natural/XLM-T/spanglish_NCM_tokenizer_XLMT_3000/tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}