Satveera commited on
Commit
3e816d7
·
verified ·
1 Parent(s): 052f86d

Upload tokenizer

Browse files
added_tokens.json CHANGED
@@ -1,3 +1,3 @@
1
  {
2
- "[PAD]": 40478
3
  }
 
1
  {
2
+ "[PAD]": 50257
3
  }
merges.txt CHANGED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json CHANGED
@@ -1,4 +1,6 @@
1
  {
 
 
2
  "pad_token": "[PAD]",
3
- "unk_token": "<unk>"
4
  }
 
1
  {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
  "pad_token": "[PAD]",
5
+ "unk_token": "<|endoftext|>"
6
  }
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,14 +1,15 @@
1
  {
 
2
  "added_tokens_decoder": {
3
- "0": {
4
- "content": "<unk>",
5
  "lstrip": false,
6
- "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
  "special": true
10
  },
11
- "40478": {
12
  "content": "[PAD]",
13
  "lstrip": false,
14
  "normalized": false,
@@ -17,9 +18,11 @@
17
  "special": true
18
  }
19
  },
 
20
  "clean_up_tokenization_spaces": true,
21
- "model_max_length": 512,
 
22
  "pad_token": "[PAD]",
23
- "tokenizer_class": "OpenAIGPTTokenizer",
24
- "unk_token": "<unk>"
25
  }
 
1
  {
2
+ "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
  "lstrip": false,
7
+ "normalized": true,
8
  "rstrip": false,
9
  "single_word": false,
10
  "special": true
11
  },
12
+ "50257": {
13
  "content": "[PAD]",
14
  "lstrip": false,
15
  "normalized": false,
 
18
  "special": true
19
  }
20
  },
21
+ "bos_token": "<|endoftext|>",
22
  "clean_up_tokenization_spaces": true,
23
+ "eos_token": "<|endoftext|>",
24
+ "model_max_length": 1024,
25
  "pad_token": "[PAD]",
26
+ "tokenizer_class": "GPT2Tokenizer",
27
+ "unk_token": "<|endoftext|>"
28
  }
vocab.json CHANGED
The diff for this file is too large to render. See raw diff