tokenizer_config.json · Compactbot/banana-idk-3m at main

banana-idk-3m / tokenizer_config.json

Add tokenizer files

f26570e verified 21 days ago

173 Bytes

{
  "model_type": "bpe",
  "bos_token": "<bos>",
  "eos_token": "<eos>",
  "unk_token": "<unk>",
  "pad_token": "<pad>",
  "add_bos_token": false,
  "add_eos_token": false
}

	{
	"model_type": "bpe",
	"bos_token": "<bos>",
	"eos_token": "<eos>",
	"unk_token": "<unk>",
	"pad_token": "<pad>",
	"add_bos_token": false,
	"add_eos_token": false
	}