Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 4,076 Bytes
4a62dcc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """eval/heldout_results.json ์ผ๋ก README.md ๋ฅผ ์ฑ์ด๋ค.
- ๋ณธ๋ฌธ์ 'ํ์ตํ์ง ์์ ๊ณต๊ฐ ๋ฒค์น๋งํฌ' ํ
- ๋งจ ์ YAML ์ model-index (๋ชจ๋ธ ํ์ด์ง ์ค๋ฅธ์ชฝ 'Evaluation results' ์์)
python eval/fill_card.py [--repo ๊ณ์ /KJH-Judge-4B-ko]
"""
import argparse
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
NAMES = {"nsmc": "NSMC", "kornli": "KorNLI (XNLI)", "klue_sts": "KLUE-STS", "unsmile": "Korean UnSmile", "kmmlu": "KMMLU", "haerae": "HAE-RAE Bench 1.0"}
META = { # model-index: (๊ณผ์ ์ข
๋ฅ, ๊ณผ์ ์ด๋ฆ, ๋ฐ์ดํฐ์
id, ์ค์ , ๋ถํ )
"nsmc": ("text-classification", "Sentiment classification", "e9t/nsmc", None, "test"),
"kornli": ("text-classification", "Natural language inference", "kakaobrain/kor_nli", "xnli", "test"),
"klue_sts": ("text-classification", "Paraphrase detection (binary STS)", "klue/klue", "sts", "validation"),
"unsmile": ("text-classification", "Hate speech detection", "smilegate-ai/kor_unsmile", None, "valid"),
"kmmlu": ("multiple-choice", "Knowledge QA", "HAERAE-HUB/KMMLU", None, "test"),
"haerae": ("multiple-choice", "Korean knowledge QA", "HAERAE-HUB/HAE_RAE_BENCH_1.0", None, "test"),
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--repo", help="๊ณ์ /์ ์ฅ์ ์ด๋ฆ. ์ฃผ๋ฉด ์นด๋ ์์ KJH/KJH-Judge-4B-ko ๋ฅผ ๋ฐ๊พผ๋ค")
a = ap.parse_args()
R = json.loads((ROOT / "eval" / "heldout_results.json").read_text(encoding="utf-8"))["results"]
card = (ROOT / "README.md").read_text(encoding="utf-8")
rows = ["| ๋ฒค์น๋งํฌ | ๋ฌด์์ ์ฌ๋ | ๋ฌธํญ | ์ ํ๋ (95% ๊ตฌ๊ฐ) | ๋ฌด์์ | ์ถ์ฒ ยท ๋ผ์ด์ ์ค |", "|---|---|---:|---:|---:|---|"]
for k, v in R.items():
rows.append(f"| {NAMES.get(k, k)} | {v['desc']} | {v['n']:,} | **{v['acc'] * 100:.1f}%** ({v['ci95'][0] * 100:.1f}~{v['ci95'][1] * 100:.1f}) | {v['chance'] * 100:.0f}% | {v['source']} ยท {v['license']} |")
s, e = card.index("<!-- HELDOUT:start -->"), card.index("<!-- HELDOUT:end -->")
card = card[:s] + "<!-- HELDOUT:start -->\n" + "\n".join(rows) + "\n" + card[e:]
# ๊ฐ์ ์กฐ๊ฑด ๋น๊ต (eval/compare/*.json, bench_compare.py ๊ฒฐ๊ณผ)
sys_path = str(ROOT / "eval")
import sys
sys.path.insert(0, sys_path)
from make_chart import comparisons
comps = comparisons()
if comps and "<!-- COMPARE:start -->" in card:
hdr = "| ๋ฒค์น๋งํฌ | **KJH-Judge-4B-ko (4B)** | " + " | ".join(n for n, _ in comps) + " |"
cmp_rows = ["**๊ฐ์ ์กฐ๊ฑด ๋น๊ต** ยท ๊ฐ์ ๋ฌธํญ ยท ๊ฐ์ ํ๋กฌํํธ ยท 0-shot ยท ๋ณด๊ธฐ ๊ธฐํธ ์ ์. ๋ค๋ฅธ ๋ชจ๋ธ์ ์๊ฐ(thinking)์ ๋๊ณ ๊ฐ์์ ๋ํ ํ์์ผ๋ก ๋ฌผ์์ต๋๋ค.", "",
hdr, "|---|---:|" + "---:|" * len(comps)]
for k, v in R.items():
cells = []
for _, c in comps:
cells.append(f"{c[k]['acc'] * 100:.1f}%" if k in c else "โ")
cmp_rows.append(f"| {NAMES.get(k, k)} | **{v['acc'] * 100:.1f}%** | " + " | ".join(cells) + " |")
s, e = card.index("<!-- COMPARE:start -->"), card.index("<!-- COMPARE:end -->")
card = card[:s] + "<!-- COMPARE:start -->\n" + "\n".join(cmp_rows) + "\n" + card[e:]
mi = ["model-index:", " - name: KJH-Judge-4B-ko", " results:"]
for k, v in R.items():
t, tn, ds, cfg, sp = META[k]
mi += [f" - task: {{type: {t}, name: {tn}}}",
f" dataset: {{name: {NAMES[k]}, type: {ds}" + (f", config: {cfg}" if cfg else "") + f", split: {sp}}}",
f" metrics:", f" - {{type: accuracy, value: {v['acc'] * 100:.1f}, name: accuracy (0-shot, n={v['n']})}}"]
head, body = card.split("\n---\n", 1)
head = re.sub(r"\nmodel-index:[\s\S]*$", "", head)
card = head + "\n" + "\n".join(mi) + "\n---\n" + body
if a.repo:
card = card.replace("KJH/KJH-Judge-4B-ko", a.repo)
(ROOT / "README.md").write_text(card, encoding="utf-8")
print("\n".join(rows))
if __name__ == "__main__":
main()
|