Quantum-Classifier / tokenizer_config.json
Nebulixlabs's picture
Upload 9 files
6641f90 verified
Raw History Blame Contribute Delete
454 Bytes
{
"tokenizer_class": "QuantumByteLevelBPE",
"model_name": "Quantum-Clasifier",
"tokenizer_type": "byte_level_bpe",
"vocab_size": 4096,
"model_max_length": 128,
"max_length": 128,
"padding_side": "right",
"truncation_side": "right",
"do_lower_case": false,
"pad_token": "<pad>",
"unk_token": "<unk>",
"cls_token": "<cls>",
"sep_token": "<sep>",
"pad_token_id": 0,
"unk_token_id": 1,
"cls_token_id": 2,
"sep_token_id": 3
}