bitadin commited on
Commit
ea2d5d8
·
verified ·
1 Parent(s): 527ab90

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +21 -3
  2. tokenizer.json +20 -4
special_tokens_map.json CHANGED
@@ -101,7 +101,25 @@
101
  "<extra_id_98>",
102
  "<extra_id_99>"
103
  ],
104
- "eos_token": "</s>",
105
- "pad_token": "<pad>",
106
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  }
 
101
  "<extra_id_98>",
102
  "<extra_id_99>"
103
  ],
104
+ "eos_token": {
105
+ "content": "</s>",
106
+ "lstrip": false,
107
+ "normalized": false,
108
+ "rstrip": false,
109
+ "single_word": false
110
+ },
111
+ "pad_token": {
112
+ "content": "<pad>",
113
+ "lstrip": false,
114
+ "normalized": false,
115
+ "rstrip": false,
116
+ "single_word": false
117
+ },
118
+ "unk_token": {
119
+ "content": "<unk>",
120
+ "lstrip": false,
121
+ "normalized": false,
122
+ "rstrip": false,
123
+ "single_word": false
124
+ }
125
  }
tokenizer.json CHANGED
@@ -1,7 +1,21 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
@@ -950,7 +964,8 @@
950
  "pre_tokenizer": {
951
  "type": "Metaspace",
952
  "replacement": "▁",
953
- "add_prefix_space": true
 
954
  },
955
  "post_processor": {
956
  "type": "TemplateProcessing",
@@ -1009,7 +1024,8 @@
1009
  "decoder": {
1010
  "type": "Metaspace",
1011
  "replacement": "▁",
1012
- "add_prefix_space": true
 
1013
  },
1014
  "model": {
1015
  "type": "Unigram",
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 89,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 89
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 0,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
  "added_tokens": [
20
  {
21
  "id": 0,
 
964
  "pre_tokenizer": {
965
  "type": "Metaspace",
966
  "replacement": "▁",
967
+ "add_prefix_space": true,
968
+ "prepend_scheme": "always"
969
  },
970
  "post_processor": {
971
  "type": "TemplateProcessing",
 
1024
  "decoder": {
1025
  "type": "Metaspace",
1026
  "replacement": "▁",
1027
+ "add_prefix_space": true,
1028
+ "prepend_scheme": "always"
1029
  },
1030
  "model": {
1031
  "type": "Unigram",