Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
!pip install -q transformers datasets evaluate seqeval ray[tune] optuna
|
| 2 |
+
|
| 3 |
+
from datasets import load_dataset
|
| 4 |
+
from transformers import AutoTokenizer, AutoModelForTokenClassification, DataCollatorForTokenClassification
|
| 5 |
+
from functools import partial
|
| 6 |
+
|
| 7 |
+
# 1. 데이터셋 및 라벨 로드
|
| 8 |
+
dataset = load_dataset("klue", "ner")
|
| 9 |
+
label_list = dataset["train"].features["ner_tags"].feature.names
|
| 10 |
+
model_name = "Leo97/KoELECTRA-small-v3-modu-ner"
|
| 11 |
+
|
| 12 |
+
# 2. 토크나이저 로드
|
| 13 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name)
|