a7dc088
1
2
3
4
5
6
7
8
9
{ "backend": "tokenizers", "bos_token": "[[ctrl:endoftext]]", "clean_up_tokenization_spaces": false, "eos_token": "[[ctrl:endoftext]]", "model_max_length": 512, "pad_token": "[[ctrl:endoftext]]", "tokenizer_class": "TokenizersBackend" }
