pratis / app.py
TOT7's picture
Create app.py
8828e4d verified
Raw History Blame Contribute Delete
516 Bytes
!pip install -q transformers datasets evaluate seqeval ray[tune] optuna
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForTokenClassification, DataCollatorForTokenClassification
from functools import partial
# 1. ๋ฐ์ดํ„ฐ์…‹ ๋ฐ ๋ผ๋ฒจ ๋กœ๋“œ
dataset = load_dataset("klue", "ner")
label_list = dataset["train"].features["ner_tags"].feature.names
model_name = "Leo97/KoELECTRA-small-v3-modu-ner"
# 2. ํ† ํฌ๋‚˜์ด์ € ๋กœ๋“œ
tokenizer = AutoTokenizer.from_pretrained(model_name)