File size: 516 Bytes
8828e4d
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
!pip install -q transformers datasets evaluate seqeval ray[tune] optuna

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForTokenClassification, DataCollatorForTokenClassification
from functools import partial

# 1. ๋ฐ์ดํ„ฐ์…‹ ๋ฐ ๋ผ๋ฒจ ๋กœ๋“œ
dataset = load_dataset("klue", "ner")
label_list = dataset["train"].features["ner_tags"].feature.names
model_name = "Leo97/KoELECTRA-small-v3-modu-ner"

# 2. ํ† ํฌ๋‚˜์ด์ € ๋กœ๋“œ
tokenizer = AutoTokenizer.from_pretrained(model_name)