colorpicker-models / tokenizer_config.json

add tokenizer_config.json (batch pad_token; required by transformers.js AutoTokenizer)

008c461 verified 12 days ago

185 Bytes

{
 "clean_up_tokenization_spaces": true,
 "model_max_length": 512,
 "pad_token": "[PAD]",
 "unk_token": "[UNK]",
 "cls_token": "[CLS]",
 "sep_token": "[SEP]",
 "mask_token": "[MASK]"
}

	{
	"clean_up_tokenization_spaces": true,
	"model_max_length": 512,
	"pad_token": "[PAD]",
	"unk_token": "[UNK]",
	"cls_token": "[CLS]",
	"sep_token": "[SEP]",
	"mask_token": "[MASK]"
	}