File size: 516 Bytes
8828e4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 | !pip install -q transformers datasets evaluate seqeval ray[tune] optuna
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForTokenClassification, DataCollatorForTokenClassification
from functools import partial
# 1. ๋ฐ์ดํฐ์
๋ฐ ๋ผ๋ฒจ ๋ก๋
dataset = load_dataset("klue", "ner")
label_list = dataset["train"].features["ner_tags"].feature.names
model_name = "Leo97/KoELECTRA-small-v3-modu-ner"
# 2. ํ ํฌ๋์ด์ ๋ก๋
tokenizer = AutoTokenizer.from_pretrained(model_name) |