{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[USER]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "[ASSISTANT]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "[SYSTEM]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": null, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "Unigram", "unk_id": 0, "vocab": [ [ "[UNK]", 0.0 ], [ "[PAD]", 0.0 ], [ "[BOS]", 0.0 ], [ "[EOS]", 0.0 ], [ "[USER]", 0.0 ], [ "[ASSISTANT]", 0.0 ], [ "[SYSTEM]", 0.0 ], [ "à®", -2.522225656791816 ], [ "à¤", -2.6667559298968397 ], [ "à¥", -2.667867464544863 ], [ "à¯", -2.693697336697005 ], [ "¯", -3.1797436685734763 ], [ "¥", -3.2232164557488874 ], [ "¤", -3.2251562253853177 ], [ "®", -3.5422916686412953 ], [ "IJ", -5.162943678772378 ], [ "ı", -5.200304987805264 ], [ "ĥ", -5.207801614211164 ], [ "¿", -5.20780161421118 ], [ "¹", -5.20780161421118 ], [ "²", -5.37537702068629 ], [ "°", -5.378459986913359 ], [ "Ĺ", -5.378459986913391 ], [ "¡", -5.378459986913398 ], [ "½", -5.37919069656 ], [ "¼", -5.37919069656 ], [ "Ĥ", -5.3995215965198415 ], [ "Ħ", -5.399521596519859 ], [ "Ĭ", -5.399521596519898 ], [ "ij", -5.399521596519953 ], [ "ķ", -5.399521596519956 ], [ "į", -5.39952159651996 ], [ "Ń", -5.399521596519963 ], [ "ª", -5.399521596519986 ], [ "§", -5.399521596519991 ], [ "¢", -5.399521596519996 ], [ "ļ", -5.39952159652 ], [ "ĺ", -5.39952159652 ], [ "Ł", -5.399521596520002 ], [ "Ľ", -5.399521596520007 ], [ "ľ", -5.399521596520014 ], [ "Ŀ", -5.399521596520014 ], [ "à", -5.706172920020648 ], [ "Ģ", -5.867563657486301 ], [ "ģ", -5.867563657486301 ], [ "»", -5.867563657486312 ], [ "¸", -5.86756365748632 ], [ "¶", -5.867563657486324 ], [ "º", -5.867563657486327 ], [ "¾", -5.867563657486327 ], [ "´", -5.867563657486329 ], [ "ħ", -5.867563657486332 ], [ "·", -5.867563657486333 ], [ "ī", -5.867563657486343 ], [ "³", -5.867563657486343 ], [ "Ī", -5.867563657486343 ], [ "ĩ", -5.867563657486347 ], [ "Ĩ", -5.867563657486347 ], [ "µ", -5.867563657486347 ], [ "±", -5.867563657486356 ], [ "ĭ", -5.8675636574863645 ], [ "Į", -5.867563657486385 ], [ "Ĵ", -5.867563657486386 ], [ "Ķ", -5.867563657486395 ], [ "ĵ", -5.867563657486395 ], [ "ĸ", -5.867563657486403 ], [ "İ", -5.867563657486404 ], [ "Ļ", -5.867563657486407 ], [ "©", -5.867563657486414 ], [ "¬", -5.867563657486414 ], [ "«", -5.867563657486414 ], [ "¨", -5.867563657486416 ], [ "¦", -5.867563657486421 ], [ "ŀ", -5.867563657486435 ], [ "ł", -5.867563657486437 ], [ "£", -5.8675636574864445 ], [ "ģà", -6.143411516194571 ], [ "Ģà", -6.143411516194573 ], [ "¾à", -6.1434115161945755 ], [ "¸à", -6.14341151619459 ], [ "»à", -6.143411516194591 ], [ "ºà", -6.143411516194591 ], [ "´à", -6.143411516194593 ], [ "µà", -6.143411516194604 ], [ "³à", -6.143411516194608 ], [ "·à", -6.143411516194613 ], [ "¶à", -6.143411516194613 ], [ "īà", -6.143411516194614 ], [ "Īà", -6.143411516194614 ], [ "ĩà", -6.143411516194614 ], [ "Ĩà", -6.1434115161946155 ], [ "ħà", -6.1434115161946155 ], [ "±à", -6.143411516194638 ], [ "ĵà", -6.143411516194652 ], [ "Ĵà", -6.143411516194652 ], [ "Ķà", -6.143411516194656 ], [ "ĭà", -6.143411516194673 ], [ "¨à", -6.143411516194675 ], [ "ĸà", -6.143411516194676 ], [ "«à", -6.143411516194679 ], [ "İà", -6.14341151619468 ], [ "Įà", -6.143411516194681 ], [ "¬à", -6.1434115161946865 ], [ "ŀà", -6.143411516194687 ], [ "¦à", -6.14341151619469 ], [ "Ļà", -6.1434115161946945 ], [ "©à", -6.143411516194696 ], [ "łà", -6.143411516194703 ], [ "£à", -6.143411516194703 ], [ "ĉ", -6.996277012105643 ], [ "Ġ", -6.996277012105643 ], [ "5", -6.996277012105654 ], [ "4", -6.996277012105654 ], [ "6", -6.996277012105654 ], [ "%", -6.996277012105677 ], [ "_", -6.996277012105677 ], [ "{", -6.996277012105677 ], [ "`", -6.996277012105677 ], [ "&", -6.996277012105677 ], [ "\\", -6.996277012105677 ], [ "@", -6.996277012105677 ], [ "(", -6.996277012105677 ], [ "'", -6.996277012105677 ], [ "+", -6.996277012105677 ], [ "#", -6.996277012105677 ], [ "|", -6.996277012105677 ], [ "[", -6.996277012105677 ], [ ")", -6.996277012105677 ], [ ">", -6.996277012105677 ], [ "?", -6.996277012105677 ], [ "]", -6.996277012105677 ], [ "*", -6.996277012105677 ], [ "^", -6.996277012105677 ], [ "=", -6.996277012105677 ], [ "-", -6.996277012105677 ], [ "$", -6.996277012105677 ], [ ",", -6.996277012105677 ], [ "\"", -6.996277012105677 ], [ "g", -6.9962770121057245 ], [ "i", -6.9962770121057245 ], [ "<", -6.9962770121057245 ], [ ".", -6.9962770121057245 ], [ ";", -6.9962770121057245 ], [ "m", -6.9962770121057245 ], [ ":", -6.9962770121057245 ], [ "e", -6.9962770121057245 ], [ "a", -6.9962770121057245 ], [ "d", -6.9962770121057245 ], [ "c", -6.9962770121057245 ], [ "l", -6.9962770121057245 ], [ "n", -6.9962770121057245 ], [ "/", -6.9962770121057245 ], [ "f", -6.9962770121057245 ], [ "h", -6.9962770121057245 ], [ "b", -6.9962770121057245 ], [ "H", -6.996277012145591 ], [ "č", -6.996277012145591 ], [ "1", -6.996277012145591 ], [ "A", -6.996277012145591 ], [ "O", -6.996277012145591 ], [ "C", -6.996277012145591 ], [ "U", -6.996277012145591 ], [ "0", -6.996277012145591 ], [ "3", -6.996277012145591 ], [ "K", -6.996277012145591 ], [ "j", -6.996277012145591 ], [ "R", -6.996277012145591 ], [ "I", -6.996277012145591 ], [ "Z", -6.996277012145591 ], [ "Č", -6.996277012145591 ], [ "z", -6.996277012145591 ], [ "F", -6.996277012145591 ], [ "~", -6.996277012145591 ], [ "X", -6.996277012145591 ], [ "r", -6.996277012145591 ], [ "u", -6.996277012145591 ], [ "B", -6.996277012145591 ], [ "M", -6.996277012145591 ], [ "S", -6.996277012145591 ], [ "s", -6.996277012145591 ], [ "V", -6.996277012145591 ], [ "Q", -6.996277012145591 ], [ "v", -6.996277012145591 ], [ "G", -6.996277012145591 ], [ "E", -6.996277012145591 ], [ "ċ", -6.996277012145591 ], [ "}", -6.996277012145591 ], [ "k", -6.996277012145591 ], [ "y", -6.996277012145591 ], [ "2", -6.996277012145591 ], [ "9", -6.996277012145591 ], [ "P", -6.996277012145591 ], [ "J", -6.996277012145591 ], [ "T", -6.996277012145591 ], [ "w", -6.996277012145591 ], [ "W", -6.996277012145591 ], [ "L", -6.996277012145591 ], [ "D", -6.996277012145591 ], [ "7", -6.996277012145591 ], [ "!", -6.996277012145591 ], [ "t", -6.996277012145591 ], [ "Y", -6.996277012145591 ], [ "8", -6.996277012145591 ], [ "x", -6.996277012145591 ], [ "p", -6.996277012145591 ], [ "q", -6.996277012145591 ], [ "Ċ", -6.996277012145591 ], [ "o", -6.996277012145591 ], [ "N", -6.996277012145591 ], [ "ľà", -7.523086418630371 ], [ "Ľà", -7.523086418630371 ], [ "ļà", -7.523086418630371 ], [ "ķà", -7.523086418630371 ], [ "ĺà", -7.523086418630388 ], [ "Ŀà", -7.5230864186304 ], [ "§à", -7.523086418630403 ], [ "Łà", -7.523086418630409 ], [ "ijà", -7.523086418630413 ], [ "¢à", -7.523086418630423 ], [ "ªà", -7.523086418630475 ], [ "Ĥà", -7.523086418630519 ], [ "Ńà", -7.523086418630532 ], [ "įà", -7.523086418630566 ], [ "Ĭà", -7.5230864186305855 ], [ "Ħà", -7.523086418630586 ], [ "½à", -7.680330088846005 ], [ "¼à", -7.68033008884604 ], [ "Ĺà", -7.686431620520356 ], [ "¡à", -7.686431620520374 ], [ "°à", -7.686431620520441 ], [ "²à", -7.712554823632267 ] ], "byte_fallback": false } }