Nschadrack commited on
Commit
0554ce9
·
verified ·
1 Parent(s): 0f4d91d

Initial alta tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +29 -0
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 65536,
3
+ "model_type": "BPE",
4
+ "byte_fallback": true,
5
+ "special_tokens": [
6
+ "<|pad|>",
7
+ "<|unk|>",
8
+ "<|bos|>",
9
+ "<|eos|>",
10
+ "<|im_start|>",
11
+ "<|im_end|>",
12
+ "<|begin_of_text|>",
13
+ "<|start_header_id|>",
14
+ "<|end_header_id|>",
15
+ "<|eot_id|>",
16
+ "<|thought_start|>",
17
+ "<|thought_end|>",
18
+ "<|tool_start|>",
19
+ "<|tool_end|>",
20
+ "<|tool_call|>",
21
+ "<|tool_result|>",
22
+ "<|fim_prefix|>",
23
+ "<|fim_middle|>",
24
+ "<|fim_suffix|>",
25
+ "<|fim_pad|>"
26
+ ],
27
+ "language": "Kinyarwanda",
28
+ "apostrophe_handling": "preserved_in_words"
29
+ }