Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download eval/bench_heldout.py from coderecode95/KJH-Judge-4B-ko: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/bench_heldout.py
- Command line
-
hf download hf://coderecode95/KJH-Judge-4B-ko/eval/bench_heldout.py
-
curl -L -o bench_heldout.py https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/bench_heldout.py
10.1 kB
| """ํ์ต์ ์ฐ์ง ์์ ๊ณต๊ฐ ํ๊ตญ์ด ๋ฒค์น๋งํฌ๋ก KJH-Judge-4B-ko ๋ฅผ ์ฐ๋ค (0-shot, ๋ณด๊ธฐ ๊ธฐํธ ์ ์ ๋ฐฉ์). | |
| # ํ์ ์๋ฒ(API)๋ก | |
| python eval/bench_heldout.py --url http://์๋ฒ:ํฌํธ | |
| # ๋ด๋ ค๋ฐ์ ๋ชจ๋ธ๋ก (GPU ๊ถ์ฅ) | |
| python eval/bench_heldout.py --model . | |
| ํ์ต ๋ฐ์ดํฐ์ ๊ฒน์น์ง ์๋ ์ถ์ฒ๋ง ์ด๋ค. ์ด ๋ชจ๋ธ์ ํ์ต ๋ฐ์ดํฐ๋ KLUE(NLIยทYNAT), KoBEST, korean-hate-speech, APEACH, SIB-200, | |
| ์์ฒด ํฉ์ฑ(์ ์ฑ ยท๋ฒ๋ นยท๋๊ตฌ ํธ์ถ)์ด๋ค. ์๋ ๋ฒค์น๋งํฌ๋ ๊ทธ์ ๋ค๋ฅธ ์ถ์ฒ๋ค. (์๋ณธ Qwen3-4B์ ์ฌ์ ํ์ต์ ๋ค์ด๊ฐ๋์ง๋ ์ ์ ์๋ค.) | |
| ํ๋ณธ์ ๊ณ ์ ์๋๋ก ๋ฝ๋๋ค. | |
| """ | |
| import argparse | |
| import concurrent.futures as cf | |
| import io | |
| import json | |
| import math | |
| import random | |
| import time | |
| import urllib.parse | |
| import urllib.request | |
| from pathlib import Path | |
| HERE = Path(__file__).resolve().parent | |
| NLI = ("์ ์ ๋ฅผ ์ฐธ์ด๋ผ๊ณ ํ ๋ ๊ฐ์ค์ ๋ถ๋ฅํ๋ค. entailment: ์ ์ ๋ก๋ถํฐ ๊ฐ์ค์ด ๋ฐ๋ผ์จ๋ค. contradiction: ์ ์ ์ ๊ฐ์ค์ด ์๋ฆฝํ ์ ์๋ค. neutral: ๋ ๋ค ์๋๋ค. " | |
| "์ ๋ณด๊ฐ ์๋ค๋ ์ด์ ๋ง์ผ๋ก contradiction์ ๊ณ ๋ฅด์ง ์๋๋ค. neutral์ ์ ์ ์ ๋ต์ด๋ฉฐ insufficient_context๊ฐ ์๋๋ค. ํต์์ ์ธ ๋ฌธ์ฅ ์๋ฏธ๋ก ํ๋จํ๋ฉฐ ํฌ๋ฐํ ๊ฐ์ ์ ๋ง๋ค์ด ๋ชจํธํ๋ค๊ณ ํ์ง ์๋๋ค. " | |
| "์ ์ ์ ์ด๊ฑฐ ๋์๊ณผ ๊ฐ์ค์ ๋ฒ์๋ฅผ ๋น ์ง์์ด ๋์กฐํ๋ค. ๊ฐ์ค์ ๋ง/๋ฐ์/์ค์ง/๋ชจ๋ ๊ฐ์ ํํ์ด ์ ์ ์ ๋ช ์๋ ๋ค๋ฅธ ๋์์ด๋ ์ฌ๋ก๋ฅผ ๋ฐฐ์ ํ๋์ง ํ์ธํ๋ค. " | |
| "์ ์ ์ ๋ฐฐํ ํํ์ด ์๋ค๋ ์ฌ์ค๋ง์ผ๋ก ์ค๋ฆฝ์ด๋ผ๊ณ ํ์ง ์๋๋ค. ํต์์ ์ธ ๋ฌธ๋งฅ์ด ๋ช ํํ๋ฉด ๋ค๋ฅธ ์์ธยท์ํฉ์ ์์๋ก ์ถ๊ฐํ์ง ์๋๋ค.") | |
| SENT = "๋ฌธ์ฅ์ ์ ๋ฐ์ ์ธ ๊ฐ์ ๊ทน์ฑ์ ํ๋จํ๋ค. ๋ถ์ ํํ๊ณผ ๋ฐ์ด๋ฅผ ๊ณ ๋ คํ๋ค." | |
| HATE = ("APEACH ์ ๋ ผ๋ฌธ์ ์์ฑ ๊ธฐ์ค์ ๋ฐ๋ฅธ๋ค. ์ฑ๋ณยท์ฑ์ ์งํฅ/์ ์ฒด์ฑยท๋์ด/์ ๋ถ/๊ฒฝ๋ ฅยท๊ตญ์ /์ธ์ข ยท์ถ์ /๊ฑฐ์ฃผ์งยท์ธ๋ชจยท์ง๋ณ/์ฅ์ ยท์์ต๊ดยทํ๋ ฅ/์ ๊ณต ๊ด๋ จ ์ฐจ๋ณยท๋นํยท๊ณต๊ฒฉ์ ํฌํจํ๋ค. " | |
| "๋ณดํธ์ง๋จ ์ ์ฒด์ฑ์ ๋ช ์ํด์ผ ํ๋ค๋ ์ ํ์ ์๋ค. ๋ถํ์ํ๊ฑฐ๋ ๊ณต๊ฒฉ์ ์ธ ์ธ๋ชจ ํ๊ฐ, ์์ต๊ด ๊ฐ์/ํํ๋ ํด๋นํ๋ค. ๋น๊ณต๊ฒฉ์ ์ธ ์ฃผ์ ์ธ๊ธ์ด๋ ์ฐจ๋ณ์ ๋ํ ๋ฐ๋ ๋ฐ์ธ์ ๋นํ์ค๋ค.") | |
| STS = "๋ ๋ฌธ์ฅ์ ์๋ฏธ๊ฐ ๊ฐ์์ง ํ๋จํ๋ค. ํํ์ด ๋ฌ๋ผ๋ ์ ๋ฌํ๋ ๋ด์ฉ์ด ๊ฐ์ผ๋ฉด ๊ฐ์ ์๋ฏธ๋ค." | |
| MCQ = "์ง๋ฌธ์ ๊ฐ์ฅ ์๋ง์ ๋ต์ ๋ณด๊ธฐ์์ ๊ณ ๋ฅธ๋ค." | |
| BENCH = { # ์ด๋ฆ: (์ค๋ช , ์ถ์ฒ, ๋ผ์ด์ ์ค) | |
| "nsmc": ("์ํ ๋ฆฌ๋ทฐ ๊ฐ์ฑ (๊ธ์ /๋ถ์ )", "e9t/nsmc ยท ratings_test", "CC0 1.0"), | |
| "kornli": ("๋ฌธ์ฅ ๊ด๊ณ 3๋ถ๋ฅ (XNLI ํ๊ตญ์ด ๋ฒ์ญ)", "kakaobrain/kor_nli ยท xnli/test", "CC BY-SA 4.0"), | |
| "klue_sts": ("๋ ๋ฌธ์ฅ์ด ๊ฐ์ ๋ป์ธ์ง (์/์๋์ค)", "klue/klue ยท sts/validation (binary-label)", "CC BY-SA 4.0"), | |
| "unsmile": ("์ ํยทํ์ค ํํ ์ฌ๋ถ", "smilegate-ai/kor_unsmile ยท valid (clean ์ฌ๋ถ)", "CC BY-NC-ND 4.0"), | |
| "kmmlu": ("ํ๊ตญ์ด ์ ๋ฌธ ์ง์ 4์ง์ ๋ค (45๊ณผ๋ชฉ)", "HAERAE-HUB/KMMLU ยท test", "CC BY-ND 4.0"), | |
| "haerae": ("ํ๊ตญ ๋ฌธํยท์ดํ ์ง์ 5์ง์ ๋ค", "HAERAE-HUB/HAE_RAE_BENCH_1.0 ยท test", "CC BY-NC-ND 4.0"), | |
| } | |
| def get(url, tries=6): | |
| url = urllib.parse.quote(url, safe=":/?=&%") # ์ค์ ์ด๋ฆ์ ๋์ด์ฐ๊ธฐ๊ฐ ์๋ ๋ฐ์ดํฐ์ (HAE-RAE) | |
| for k in range(tries): | |
| try: | |
| return urllib.request.urlopen(url, timeout=120).read() | |
| except Exception: | |
| if k == tries - 1: | |
| raise | |
| time.sleep(3) | |
| def parquet(dataset, config, split): | |
| import pyarrow.parquet as pq | |
| urls = json.loads(get(f"https://huggingface.co/api/datasets/{dataset}/parquet"))[config][split] | |
| rows = [] | |
| for u in urls: | |
| rows += pq.read_table(io.BytesIO(get(u))).to_pylist() | |
| return rows | |
| def load(name, rng, n): | |
| """โ [(์์ฒญ, ์ ๋ต ๋ฒํธ)]""" | |
| out = [] | |
| if name == "nsmc": | |
| lines = get("https://raw.githubusercontent.com/e9t/nsmc/master/ratings_test.txt").decode("utf-8").splitlines()[1:] | |
| rows = [l.split("\t") for l in lines] | |
| rows = [r for r in rows if len(r) == 3 and r[1].strip()] | |
| for _, doc, lab in rng.sample(rows, n): | |
| out.append(({"instruction": SENT, "item": {"text": doc}, "choices": ["๊ธ์ ์ ๊ฐ์ ", "๋ถ์ ์ ๊ฐ์ "]}, 0 if lab == "1" else 1)) | |
| elif name == "kornli": | |
| rows = parquet("kakaobrain/kor_nli", "xnli", "test") | |
| gmap = {0: 0, 1: 2, 2: 1} # ๋ฐ์ดํฐ: 0 ํจ์ 1 ์ค๋ฆฝ 2 ๋ชจ์ โ ๋ณด๊ธฐ: entailment contradiction neutral | |
| for r in rng.sample(rows, n): | |
| out.append(({"instruction": NLI, "item": {"premise": r["premise"], "hypothesis": r["hypothesis"]}, "choices": ["entailment", "contradiction", "neutral"]}, gmap[r["label"]])) | |
| elif name == "klue_sts": | |
| for r in parquet("klue/klue", "sts", "validation"): | |
| out.append(({"instruction": STS, "item": {"sentence1": r["sentence1"], "sentence2": r["sentence2"]}, "choices": ["๊ฐ์ ์๋ฏธ", "๋ค๋ฅธ ์๋ฏธ"]}, 0 if r["labels"]["binary-label"] == 1 else 1)) | |
| elif name == "unsmile": | |
| for r in rng.sample(parquet("smilegate-ai/kor_unsmile", "default", "valid"), n): | |
| out.append(({"instruction": HATE, "item": {"text": r["๋ฌธ์ฅ"]}, "choices": ["๋นํ์ค: ๋นํยท๊ณต๊ฒฉ์ฑ์ด ์๋ ํํ", "ํ์คยท๋นํยท๊ณต๊ฒฉ์ ํํ"]}, 0 if r["clean"] == 1 else 1)) | |
| elif name == "kmmlu": | |
| cfgs = json.loads(get("https://huggingface.co/api/datasets/HAERAE-HUB/KMMLU/parquet")) | |
| per = max(1, n // len(cfgs)) | |
| for c in sorted(cfgs): | |
| rows = parquet("HAERAE-HUB/KMMLU", c, "test") | |
| for r in rng.sample(rows, min(per, len(rows))): | |
| out.append(({"instruction": MCQ, "item": {"question": r["question"]}, "choices": [str(r[k]) for k in "ABCD"]}, int(r["answer"]) - 1)) | |
| elif name == "haerae": | |
| cfgs = json.loads(get("https://huggingface.co/api/datasets/HAERAE-HUB/HAE_RAE_BENCH_1.0/parquet")) | |
| per = max(1, n // len(cfgs)) | |
| for c in sorted(cfgs): | |
| rows = parquet("HAERAE-HUB/HAE_RAE_BENCH_1.0", c, "test") | |
| for r in rng.sample(rows, min(per, len(rows))): | |
| ks = [k for k in "abcde" if r.get(k) not in (None, "")] | |
| gold = str(r["answer"]).strip().lower() | |
| if gold not in ks: | |
| continue | |
| out.append(({"instruction": MCQ, "item": {"question": r["question"]}, "choices": [str(r[k]) for k in ks]}, ks.index(gold))) | |
| return out | |
| def wilson(k, n, z=1.96): | |
| if n == 0: | |
| return (0, 0) | |
| p = k / n | |
| d = 1 + z * z / n | |
| c = (p + z * z / (2 * n)) / d | |
| h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d | |
| return (c - h, c + h) | |
| def run_api(url, reqs): | |
| def post(path, body): | |
| for k in range(12): | |
| try: | |
| req = urllib.request.Request(url + path, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"}) | |
| return json.loads(urllib.request.urlopen(req, timeout=300).read()) | |
| except urllib.error.HTTPError as e: | |
| if e.code == 404: | |
| return None | |
| raise | |
| except Exception: | |
| time.sleep(2) | |
| raise RuntimeError("์๋ฒ์ ์ฐ๊ฒฐํ์ง ๋ชปํ์ต๋๋ค") | |
| wire = [{"instruction": r["instruction"], "item": r["item"], "choices": [{"text": c} for c in r["choices"]]} for r in reqs] | |
| first = post("/api/batch", {"items": wire[:32]}) | |
| if first: | |
| res = first["results"] | |
| for s in range(32, len(wire), 32): | |
| res += post("/api/batch", {"items": wire[s:s + 32]})["results"] | |
| else: | |
| with cf.ThreadPoolExecutor(8) as ex: | |
| res = list(ex.map(lambda b: post("/api/decide", b), wire)) | |
| return [ord(r["choice"]) - 65 for r in res], [r["confidence"] for r in res] | |
| def run_local(path, reqs): | |
| import sys | |
| sys.path.insert(0, str(Path(path).resolve())) | |
| from judge import Judge | |
| j = Judge(path) | |
| out = j.decide_batch(reqs) | |
| return [o["index"] for o in out], [o["confidence"] for o in out] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--url") | |
| ap.add_argument("--model") | |
| ap.add_argument("--n", type=int, default=1000, help="๋ฒค์น๋งํฌ๋ง๋ค ์ต๋ ๋ฌธํญ ์ (KLUE-STS๋ ์ ์ฒด 519)") | |
| ap.add_argument("--only", nargs="*") | |
| ap.add_argument("--out", default=str(HERE / "heldout_results.json")) | |
| a = ap.parse_args() | |
| prev = Path(a.out) | |
| summary = json.loads(prev.read_text(encoding="utf-8"))["results"] if prev.is_file() and a.only else {} # --only ๋ก ์ผ๋ถ๋ง ๋ค์ ๋๋ฆฌ๋ฉด ๋๋จธ์ง ๊ฒฐ๊ณผ๋ ์ ์ง | |
| for name in (a.only or BENCH): | |
| rng = random.Random(f"kjh-judge:{name}") | |
| t0 = time.time() | |
| data = load(name, rng, a.n) | |
| reqs, gold = [d[0] for d in data], [d[1] for d in data] | |
| pred, conf = run_api(a.url.rstrip("/"), reqs) if a.url else run_local(a.model, reqs) | |
| k = sum(p == g for p, g in zip(pred, gold)) | |
| chance = sum(1 / len(r["choices"]) for r in reqs) / len(reqs) | |
| lo, hi = wilson(k, len(reqs)) | |
| summary[name] = {"desc": BENCH[name][0], "source": BENCH[name][1], "license": BENCH[name][2], "n": len(reqs), | |
| "acc": round(k / len(reqs), 4), "ci95": [round(lo, 4), round(hi, 4)], "chance": round(chance, 4), | |
| "mean_conf": round(sum(conf) / len(conf), 4), "seconds": round(time.time() - t0)} | |
| print(f"{name:<10} n={len(reqs):>5} ์ ํ๋ {k / len(reqs):6.1%} (95% {lo:.1%}~{hi:.1%}) ๋ฌด์์ {chance:.1%} {round(time.time() - t0)}s", flush=True) | |
| Path(a.out).write_text(json.dumps({"made": time.strftime("%Y-%m-%d %H:%M"), "protocol": "0-shot, ๋ณด๊ธฐ ๊ธฐํธ ์ ์(์จ๋ ๋ณด์ softmax), ํ๋ณธ ๊ณ ์ ์๋", "results": summary}, ensure_ascii=False, indent=1), encoding="utf-8") | |
| print("์ ์ฅ:", a.out) | |
| if __name__ == "__main__": | |
| main() | |