File size: 1,856 Bytes
3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 1f87284 3919851 a6951b9 1f87284 a6951b9 46ef4d8 1f87284 a6951b9 3919851 a6951b9 3919851 a6951b9 3919851 a6951b9 3919851 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | {
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[رابط]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"6": {
"content": "[بريد]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"7": {
"content": "[مستخدم]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
}
},
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"do_basic_tokenize": true,
"do_lower_case": false,
"extra_special_tokens": {},
"mask_token": "[MASK]",
"max_len": 512,
"model_max_length": 512,
"never_split": [
"[بريد]",
"[مستخدم]",
"[رابط]"
],
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"strip_accents": null,
"tokenize_chinese_chars": true,
"tokenizer_class": "BertTokenizer",
"unk_token": "[UNK]"
}
|