L16-V65536 tokenizer with Llama-3 chat specials in reserved ids 4-8

1e037db verified about 1 month ago

156 Bytes

{
  "source_spm": "L16_V65536/spm.model",
  "vocab_size": 65536,
  "token_size_in_bytes": 2,
  "eos_token_id": 2,
  "bos_token_id": 1,
  "pad_token_id": 3
}

	{
	"source_spm": "L16_V65536/spm.model",
	"vocab_size": 65536,
	"token_size_in_bytes": 2,
	"eos_token_id": 2,
	"bos_token_id": 1,
	"pad_token_id": 3
	}