Arabic
arabic
tokenizer
sentencepiece
unigram
aratoken
mariklolik commited on
Commit
d360a54
·
verified ·
1 Parent(s): ebb2fac

Add fw2hq_wp_keep

Browse files
sentencepiece/fw2hq_wp_keep/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
sentencepiece/fw2hq_wp_keep/tokenizer_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "ArabicQwenTokenizer",
3
+ "tokenizer_file": "/home/mekashirskiy/airi/ArabicLLM/tokenizer/data/tokenizers/fw2hq_wp_keep/tokenizer.json",
4
+ "retain_diacritics": true,
5
+ "script_variant": "arabic",
6
+ "algorithm": "wordpiece",
7
+ "add_prefix_space": true,
8
+ "clean_up_tokenization_spaces": false,
9
+ "normalizer_spec": {
10
+ "unicode_normalization": "NFKC",
11
+ "remove_diacritics": false,
12
+ "lowercase_latin": true,
13
+ "preserve_alif_variants": false,
14
+ "arabic_char_replacements": {}
15
+ },
16
+ "pretokenizer_spec": {
17
+ "with_prefix_space": false,
18
+ "add_prefix_space": true,
19
+ "fuse_numeric_sequences": true
20
+ },
21
+ "model_max_length": 131072,
22
+ "pad_token": "<|endoftext|>",
23
+ "eos_token": "<|endoftext|>",
24
+ "additional_special_tokens": [
25
+ "<|endoftext|>",
26
+ "<|im_start|>",
27
+ "<|im_end|>",
28
+ "<|object_ref_start|>",
29
+ "<|object_ref_end|>",
30
+ "<|box_start|>",
31
+ "<|box_end|>",
32
+ "<|quad_start|>",
33
+ "<|quad_end|>",
34
+ "<|vision_start|>",
35
+ "<|vision_end|>",
36
+ "<|vision_pad|>",
37
+ "<|image_pad|>",
38
+ "<|video_pad|>",
39
+ "<tool_call>",
40
+ "</tool_call>",
41
+ "<|fim_prefix|>",
42
+ "<|fim_middle|>",
43
+ "<|fim_suffix|>",
44
+ "<|fim_pad|>",
45
+ "<|repo_name|>",
46
+ "<|file_sep|>"
47
+ ]
48
+ }