Hugging Face's logo

jinwoo1126
/
yi-ko-6b-text2sql

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Model card Files
xet
 Community
yi-ko-6b-text2sql / tokenizer_config.json
jinwoo1126's picture
jinwoo1126
Upload tokenizer
4755f5a verified
Raw History Blame Contribute Delete
9.57 kB
{
"add_bos_token": true,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78413": {
"content": "<|sep|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"78414": {
"content": "<|acc|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"78415": {
"content": "<|rrn|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"78416": {
"content": "<|tel|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"78417": {
"content": "γ„±",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78418": {
"content": "γ„΄",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78419": {
"content": "γ„·",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78420": {
"content": "γ„Ή",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78421": {
"content": "ㅁ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78422": {
"content": "γ…‚",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78423": {
"content": "γ……",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78424": {
"content": "γ…‡",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78425": {
"content": "γ…ˆ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78426": {
"content": "γ…Š",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78427": {
"content": "γ…‹",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78428": {
"content": "γ…Œ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78429": {
"content": "ㅍ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78430": {
"content": "γ…Ž",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78431": {
"content": "ㅏ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78432": {
"content": "ㅐ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78433": {
"content": "γ…‘",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78434": {
"content": "γ…’",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78435": {
"content": "γ…“",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78436": {
"content": "γ…”",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78437": {
"content": "γ…•",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78438": {
"content": "γ…–",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78439": {
"content": "γ…—",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78440": {
"content": "γ…˜",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78441": {
"content": "γ…™",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78442": {
"content": "γ…š",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78443": {
"content": "γ…›",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78444": {
"content": "γ…œ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78445": {
"content": "ㅝ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78446": {
"content": "γ…ž",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78447": {
"content": "γ…Ÿ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78448": {
"content": "γ… ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78449": {
"content": "γ…‘",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78450": {
"content": "γ…’",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78451": {
"content": "γ…£",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78452": {
"content": "γ„²",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78453": {
"content": "γ„³",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78454": {
"content": "γ„΅",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78455": {
"content": "γ„Ά",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78456": {
"content": "γ„Ί",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78457": {
"content": "γ„»",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78458": {
"content": "γ„Ό",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78459": {
"content": "γ„½",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78460": {
"content": "γ„Ύ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78461": {
"content": "γ„Ώ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78462": {
"content": "γ…€",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78463": {
"content": "γ…„",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|startoftext|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"fast": true,
"legacy": true,
"model_max_length": 4096,
"pad_token": "<|endoftext|>",
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": true
}