README.md

metadata

language:
  - ko

origin tokenizer : 42dot/42dot_LLM-PLM-1.3B (42dot/42dot_LLM-PLM-1.3B 의 토크나이저와 호환됩니다)

new added special tokens = ["$~me$", "$~the_flow_of_thought$", "$~dev$", "$~somebody$"]
replaced special tokens = {
  "bos_token":"$~bos$",
  "eos_token":"$~eos$",
  "pad_token":"$~pad$",
  "unk_token":"$~unk$",
}