neo-coder-v0.2 / tokenizer_config.json

praga2008

Initial release of NEO-CODER v0.2.1 (3.8B Autonomous Coding Agent)

bae47d1 verified 2 months ago

292 Bytes

{
  "add_bos_token": true,
  "add_eos_token": false,
  "bos_token": "<|im_start|>",
  "eos_token": "<|im_end|>",
  "pad_token": "<|pad|>",
  "unk_token": "<|unk|>",
  "model_max_length": 4096,
  "tokenizer_class": "PreTrainedTokenizerFast",
  "clean_up_tokenization_spaces": false
}

	{
	"add_bos_token": true,
	"add_eos_token": false,
	"bos_token": "<\|im_start\|>",
	"eos_token": "<\|im_end\|>",
	"pad_token": "<\|pad\|>",
	"unk_token": "<\|unk\|>",
	"model_max_length": 4096,
	"tokenizer_class": "PreTrainedTokenizerFast",
	"clean_up_tokenization_spaces": false
	}