Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download eval/make_chart.py from coderecode95/KJH-Judge-4B-ko: direct link, hf CLI and curl.
- Browser
- Download file 7.49 kB
-
https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/make_chart.py
- Command line
-
hf download hf://coderecode95/KJH-Judge-4B-ko/eval/make_chart.py
-
curl -L -o make_chart.py https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/make_chart.py
7.49 kB
| """๋ชจ๋ธ ์นด๋์ฉ ๋ฒค์น๋งํฌ ๊ทธ๋ฆผ(assets/benchmark.png)์ ๋ง๋ ๋ค: HTML๋ก ๊ทธ๋ฆฐ ๋ค Edge/Chrome ํค๋๋ฆฌ์ค๋ก ์ฐ๋๋ค. | |
| python eval/make_chart.py [--browser ๊ฒฝ๋ก] | |
| """ | |
| import argparse | |
| import json | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| NAMES = {"nsmc": ("NSMC", "์ํ ๋ฆฌ๋ทฐ ๊ฐ์ฑ"), "klue_sts": ("KLUE-STS", "๋ ๋ฌธ์ฅ์ด ๊ฐ์ ๋ป์ธ๊ฐ"), "unsmile": ("Korean UnSmile", "์ ํยทํ์ค ํํ ํ๋ณ"), | |
| "kornli": ("KorNLI", "๋ฌธ์ฅ ๊ด๊ณ 3๋ถ๋ฅ"), "haerae": ("HAE-RAE 1.0", "ํ๊ตญ ๋ฌธํยท์ดํ ์ง์"), "kmmlu": ("KMMLU", "์ ๋ฌธ ์ง์ 4์ง์ ๋ค")} | |
| KNOWLEDGE = {"kmmlu", "haerae"} | |
| def speed_box(): | |
| f = ROOT / "eval" / "speed_results.json" | |
| if not f.is_file(): | |
| return '<div class="box"><b>~75ms</b><span>ํ์ ํ ๋ฒ (GPU, ์งง์ ์ ๋ ฅ)<br>๊ธ์ ์์ฑํ์ง ์๊ณ ํ ๋ฒ๋ง ๊ณ์ฐ</span></div>' | |
| s = json.loads(f.read_text(encoding="utf-8")) | |
| return (f'<div class="box"><b>{s["kjh_single"]["median_ms"]}ms</b><span>ํ์ ํ ๋ฒ (์ค์๊ฐ, ์๋ณต ํฌํจ)<br>' | |
| f'Gemma 4 26B-A4B {s["api_single"]["median_ms"]}ms ยท ์๊ฐ ์ผ๊ณ ๋ตํ๋ฉด {s["api_thinking"]["median_ms"] / 1000:.1f}์ด</span></div>') | |
| def comparisons(): | |
| """eval/compare/*.json (bench_compare.py ๊ฒฐ๊ณผ) โ [(์ด๋ฆ, ๊ฒฐ๊ณผ)]. KJH ์์ (-jev)์ ๋บ๋ค.""" | |
| d = ROOT / "eval" / "compare" | |
| labels = json.loads((d / "labels.json").read_text(encoding="utf-8")) if (d / "labels.json").is_file() else {} | |
| out = [] | |
| for f in sorted(d.glob("*.json")): | |
| if f.name == "labels.json" or f.stem.endswith("-jev"): | |
| continue | |
| j = json.loads(f.read_text(encoding="utf-8")) | |
| out.append((labels.get(f.stem, j.get("model", f.stem)), j["results"])) | |
| return out | |
| def html(R): | |
| order = [k for k in NAMES if k in R] | |
| comps = comparisons() | |
| rows = "" | |
| for k in order: | |
| v, (nm, desc) = R[k], NAMES[k] | |
| acc, lo, hi, ch = v["acc"] * 100, v["ci95"][0] * 100, v["ci95"][1] * 100, v["chance"] * 100 | |
| cls = "know" if k in KNOWLEDGE else "" | |
| cmp_html = "".join(f'<div class="cmp" style="width:{c[k]["acc"] * 100}%"><i>{c[k]["acc"] * 100:.1f}</i></div>' for _, c in comps if k in c) | |
| rows += f"""<div class="row {cls} {'two' if cmp_html else ''}"><div class="nm"><b>{nm}</b><span>{desc}</span></div> | |
| <div class="trk"><div class="track"><div class="bar" style="width:{acc}%"></div><div class="ci" style="left:{lo}%;width:{hi - lo}%"></div> | |
| <div class="chance" style="left:{ch}%"><i>๋ฌด์์ {ch:.0f}%</i></div></div>{cmp_html}</div><div class="val">{acc:.1f}<small>%</small></div></div>""" | |
| legend_cmp = "".join(f'<span><i style="background:#cbd5e1;border:1px solid #94a3b8"></i>{n}</span>' for n, _ in comps) | |
| return f"""<!doctype html><html><head><meta charset="utf-8"> | |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans+KR:wght@400;500;600;700&family=IBM+Plex+Mono:wght@500;600&display=swap"> | |
| <style> | |
| *{{box-sizing:border-box}}body{{margin:0;background:#ffffff;font-family:'IBM Plex Sans KR','Malgun Gothic',sans-serif;color:#121823}} | |
| .wrap{{width:1200px;padding:40px 48px 36px}} | |
| h1{{margin:0;font-size:30px;font-weight:700;letter-spacing:-.02em}}h1 span{{color:#1d4ed8}} | |
| .sub{{margin:8px 0 28px;color:#5a6476;font-size:16px}} | |
| .grid{{display:grid;grid-template-columns:1fr 270px;gap:36px}} | |
| .row{{display:grid;grid-template-columns:210px 1fr 96px;gap:16px;align-items:center;height:62px}} | |
| .nm b{{display:block;font-size:17px}}.nm span{{font-size:13px;color:#5a6476}} | |
| .track{{position:relative;height:26px;background:#eef1f5;border-radius:5px}} | |
| .bar{{position:absolute;inset:0 auto 0 0;background:#1d4ed8;border-radius:5px}} | |
| .know .bar{{background:#94a3b8}} | |
| .ci{{position:absolute;top:11px;height:4px;background:rgba(255,255,255,.75);border-radius:2px}} | |
| .chance{{position:absolute;top:-6px;bottom:-6px;border-left:2px dashed #121823;opacity:.55}} | |
| .chance i{{position:absolute;top:-17px;left:4px;font-style:normal;font-size:11px;white-space:nowrap;color:#121823}} | |
| .val{{font:600 26px 'IBM Plex Mono',monospace;text-align:right}}.val small{{font-size:15px;color:#5a6476}} | |
| .sep{{margin:6px 0 4px;font-size:12.5px;color:#5a6476;padding-left:226px}} | |
| .side{{display:flex;flex-direction:column;gap:12px}} | |
| .box{{border:1px solid #d8dde5;border-radius:10px;padding:14px 16px}} | |
| .box b{{display:block;font:600 28px 'IBM Plex Mono',monospace;letter-spacing:-.02em}}.box span{{font-size:13px;color:#5a6476;line-height:1.45;display:block}} | |
| .foot{{margin-top:22px;font-size:12.5px;color:#5a6476}} | |
| .legend{{display:flex;flex-wrap:wrap;gap:6px 18px;font-size:13px;color:#5a6476;margin-bottom:10px;padding-left:226px}}.legend i{{display:inline-block;width:12px;height:12px;border-radius:3px;margin-right:6px;vertical-align:-1px}} | |
| .row.two{{height:78px}}.trk{{display:flex;flex-direction:column;gap:5px}} | |
| .cmp{{position:relative;height:10px;background:#cbd5e1;border-radius:3px;max-width:100%}} | |
| .cmp i{{position:absolute;left:calc(100% + 6px);top:-4px;font:500 11.5px 'IBM Plex Mono',monospace;color:#5a6476;font-style:normal}} | |
| </style></head><body><div class="wrap"> | |
| <h1><span>KJH-Judge-4B-ko</span> ยท ํ์ตํ์ง ์์ ํ๊ตญ์ด ๋ฒค์น๋งํฌ</h1> | |
| <div class="sub">ํ์ต ๋ฐ์ดํฐ์ ๋ค๋ฅธ ์ถ์ฒ์ ๊ณต๊ฐ ๋ฐ์ดํฐ, 0-shot, ๋ณด๊ธฐ ๊ธฐํธ ์ ์ ๋ฐฉ์. ๋ง๋ ์ ํฐ ์ ์ 95% ์ ๋ขฐ๊ตฌ๊ฐ.</div> | |
| <div class="grid"><div> | |
| <div class="legend"><span><i style="background:#1d4ed8"></i>KJH-Judge-4B-ko ยท ํ์ ๊ณผ์ </span><span><i style="background:#94a3b8"></i>KJH-Judge-4B-ko ยท ์ง์ ์ํ</span>{legend_cmp}</div> | |
| {rows}</div> | |
| <div class="side"> | |
| <div class="box"><b>4.02B</b><span>ํ๋ผ๋ฏธํฐ ยท Qwen3-4B์์ ๋ฏธ์ธ์กฐ์ </span></div> | |
| {speed_box()} | |
| <div class="box"><b>0.027</b><span>๊ธฐ๋ ๋ณด์ ์ค์ฐจ(ECE)<br>ํ๋ฅ ์ ๊ทธ๋๋ก ๊ธฐ์ค์ผ๋ก ์ธ ์ ์์</span></div> | |
| <div class="box"><b>97.9%</b><span>๋ณด๊ธฐ๋ฅผ ๊ฑฐ๊พธ๋ก ๋์๋ ๊ฐ์ ๋ต</span></div> | |
| </div></div> | |
| <div class="foot">๋ฌธํญ ์: NSMCยทKorNLIยทUnSmile ๊ฐ 1,000 (๊ณ ์ ์๋ ํ๋ณธ), KLUE-STS 519 (์ ์ฒด), KMMLU {R.get('kmmlu', {}).get('n', 0):,} (45๊ณผ๋ชฉ ๊ณ ๋ฅด๊ฒ), HAE-RAE {R.get('haerae', {}).get('n', 0):,} (6์์ญ ๊ณ ๋ฅด๊ฒ). ์ฌํ: eval/bench_heldout.py</div> | |
| </div></body></html>""" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--browser", default=next((p for p in [r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe", r"C:\Program Files\Google\Chrome\Application\chrome.exe", | |
| shutil.which("google-chrome") or "", shutil.which("chromium") or ""] if p and Path(p).exists()), None)) | |
| a = ap.parse_args() | |
| R = json.loads((ROOT / "eval" / "heldout_results.json").read_text(encoding="utf-8"))["results"] | |
| (ROOT / "assets").mkdir(exist_ok=True) | |
| src = ROOT / "eval" / "chart.html" | |
| src.write_text(html(R), encoding="utf-8") | |
| out = ROOT / "assets" / "benchmark.png" | |
| h = 250 + (78 if comparisons() else 62) * len(R) + 40 | |
| with tempfile.TemporaryDirectory() as prof: | |
| subprocess.run([a.browser, "--headless=new", "--disable-gpu", f"--user-data-dir={prof}", "--hide-scrollbars", "--force-device-scale-factor=2", | |
| f"--window-size=1200,{h}", "--virtual-time-budget=4000", f"--screenshot={out}", src.as_uri()], check=True, timeout=120) | |
| print("์ ์ฅ:", out) | |
| if __name__ == "__main__": | |
| main() | |