Nschadrack commited on
Commit
4cd8af4
·
verified ·
1 Parent(s): 2525ed7

Delete tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -30
tokenizer_config.json DELETED
@@ -1,30 +0,0 @@
1
- {
2
- "vocab_size": 32276,
3
- "model_type": "BPE",
4
- "byte_fallback": true,
5
- "tokenizer_class": "PreTrainedTokenizerFast",
6
- "special_tokens": [
7
- "<|pad|>",
8
- "<|unk|>",
9
- "<|bos|>",
10
- "<|eos|>",
11
- "<|im_start|>",
12
- "<|im_end|>",
13
- "<|begin_of_text|>",
14
- "<|start_header_id|>",
15
- "<|end_header_id|>",
16
- "<|eot_id|>",
17
- "<|thought_start|>",
18
- "<|thought_end|>",
19
- "<|tool_start|>",
20
- "<|tool_end|>",
21
- "<|tool_call|>",
22
- "<|tool_result|>",
23
- "<|fim_prefix|>",
24
- "<|fim_middle|>",
25
- "<|fim_suffix|>",
26
- "<|fim_pad|>"
27
- ],
28
- "language": "Kinyarwanda",
29
- "apostrophe_handling": "preserved_in_words"
30
- }