d1-browser-decision-fp16 / tokenizer_config.json

FP16-weight package of d1-browser-decision-fp32 (FP32 arithmetic)

d83c32b verified 5 days ago

259 Bytes

{
  "backend": "tokenizers",
  "bos_token": "<|startoftext|>",
  "eos_token": "<|im_end|>",
  "pad_token": "<|pad|>",
  "mask_token": "<|mask|>",
  "clean_up_tokenization_spaces": false,
  "model_max_length": 16384,
  "tokenizer_class": "TokenizersBackend"
}

	{
	"backend": "tokenizers",
	"bos_token": "<\|startoftext\|>",
	"eos_token": "<\|im_end\|>",
	"pad_token": "<\|pad\|>",
	"mask_token": "<\|mask\|>",
	"clean_up_tokenization_spaces": false,
	"model_max_length": 16384,
	"tokenizer_class": "TokenizersBackend"
	}