File size: 368 Bytes
f658a85
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
{
  "backend": "tokenizers",
  "bos_token": "<|hy_begin▁of▁sentence|>",
  "clean_up_tokenization_spaces": false,
  "eos_token": "<eos:6124c78e>",
  "local_files_only": false,
  "model_max_length": 262144,
  "pad_token": "<|hy_▁pad▁|>",
  "tokenizer_class": "TokenizersBackend",
  "bos_token_id": 120000,
  "eos_token_id": [
    120025,
    120026
  ]
}