| { |
| "model_type": "llama", |
| "tokenizer_class": "PreTrainedTokenizerFast", |
| "bos_token": "<s>", |
| "eos_token": "</s>", |
| "unk_token": "<unk>", |
| "pad_token": "<pad>", |
| "model_max_length": 512, |
| "padding_side": "right", |
| "truncation_side": "right", |
| "add_bos_token": true, |
| "add_eos_token": true, |
| "arabic_dialect_tokens": [ |
| "โุดูููู", |
| "โูุงูุฏ", |
| "โฺุฐู", |
| "โูุจู", |
| "โูุงูุญูู", |
| "โุฒูู", |
| "โฺุงู", |
| "โููุด", |
| "โุดููู", |
| "โู
ุง_ุนูุฏู" |
| ], |
| "special_tokens": { |
| "task_tokens": [ |
| "[DIALECT_DETECT]", |
| "[SARCASM_DETECT]", |
| "[NER_REGIONAL]", |
| "[PII_REDACT]" |
| ], |
| "label_tokens": [ |
| "[kuwaiti]", |
| "[gulf_gcc]", |
| "[msa]", |
| "[egyptian]", |
| "[sarcastic]", |
| "[literal]", |
| "[ambiguous]", |
| "[CIVIL_ID_REDACTED]", |
| "[PHONE_REDACTED]", |
| "[NAME_REDACTED]", |
| "[ADDRESS_REDACTED]" |
| ] |
| } |
| } |
