add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"파미오": 32009, "핀다": 32027, "제주맥주": 32001, "D램익스체인지": 32024, "끌림벤처스": 32018, "휴마시스": 32003, "이그잭스": 32002, "마그나": 32019, "픽사": 32025, "보령컨슈머헬스케어": 32023, "모더나": 32004, "큐라클": 32010, "바이오젠": 32000, "GE헬스케어": 32030, "블루런벤처스": 32021, "쏘카": 32029, "아이도트": 32012, "삼성바이오로직스": 32005, "우드맥킨지": 32007, "휴온스바이오파마": 32028, "비보": 32026, "우드매킨지": 32008, "삼성정밀화학": 32014, "야놀자": 32017, "웅진북센": 32006, "맥스트": 32015, "한미글로벌": 32016, "신한금융지주": 32020, "무신사": 32011, "카뱅": 32022, "가우리안": 32013}
+{"D램익스체인지": 32024, "끌림벤처스": 32018, "야놀자": 32017, "한미글로벌": 32016, "GE헬스케어": 32030, "아이도트": 32012, "삼성정밀화학": 32014, "보령컨슈머헬스케어": 32023, "맥스트": 32015, "웅진북센": 32006, "블루런벤처스": 32021, "마그나": 32019, "휴온스바이오파마": 32028, "쏘카": 32029, "무신사": 32011, "휴마시스": 32003, "우드맥킨지": 32007, "큐라클": 32010, "이그잭스": 32002, "삼성바이오로직스": 32005, "모더나": 32004, "가우리안": 32013, "비보": 32026, "픽사": 32025, "파미오": 32009, "핀다": 32027, "바이오젠": 32000, "제주맥주": 32001, "카뱅": 32022, "신한금융지주": 32020, "우드매킨지": 32008}

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "additional_special_tokens": ["바이오젠", "제주맥주", "이그잭스", "휴마시스", "모더나", "삼성바이오로직스", "웅진북센", "우드맥킨지", "우드매킨지", "파미오", "큐라클", "무신사", "아이도트", "가우리안", "삼성정밀화학", "맥스트", "삼성전기", "교보문고", "한미글로벌", "야놀자", "끌림벤처스", "마그나", "신한금융지주", "블루런벤처스", "카뱅", "보령컨슈머헬스케어", "D램익스체인지", "픽사", "비보", "핀다", "휴온스바이오파마", "쏘카", "GE헬스케어"]}

tokenizer_config.json

@@ -1 +1 @@
-{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "do_basic_tokenize": true, "never_split": null, "bos_token": "[CLS]", "eos_token": "[SEP]", "model_max_length": 512, "special_tokens_map_file": "/home/roa/.cache/huggingface/transformers/1a24ab4628028ed80dea35ce3334a636dc656fd9a17a09bad377f88f0cbecdac.70c17d6e4d492c8f24f5bb97ab56c7f272e947112c6faf9dd846da42ba13eb23", "name_or_path": "klue/roberta-large", "tokenizer_class": "BertTokenizer"}
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "do_basic_tokenize": true, "never_split": null, "bos_token": "[CLS]", "eos_token": "[SEP]", "model_max_length": 512, "additional_special_tokens": ["바이오젠", "제주맥주", "이그잭스", "휴마시스", "모더나", "삼성바이오로직스", "웅진북센", "우드맥킨지", "우드매킨지", "파미오", "큐라클", "무신사", "아이도트", "가우리안", "삼성정밀화학", "맥스트", "삼성전기", "교보문고", "한미글로벌", "야놀자", "끌림벤처스", "마그나", "신한금융지주", "블루런벤처스", "카뱅", "보령컨슈머헬스케어", "D램익스체인지", "픽사", "비보", "핀다", "휴온스바이오파마", "쏘카", "GE헬스케어"], "special_tokens_map_file": "/home/roa/.cache/huggingface/transformers/1a24ab4628028ed80dea35ce3334a636dc656fd9a17a09bad377f88f0cbecdac.70c17d6e4d492c8f24f5bb97ab56c7f272e947112c6faf9dd846da42ba13eb23", "name_or_path": "klue/roberta-large", "tokenizer_class": "BertTokenizer"}