Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 9,089 Bytes
4a62dcc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | """๋ค๋ฅธ ๋ชจ๋ธ๊ณผ ๊ฐ์ ์กฐ๊ฑด์ผ๋ก ๋น๊ต: bench_heldout.py ์ ๊ฐ์ ๋ฐ์ดํฐ ยท ๊ฐ์ ํ๋กฌํํธ ยท ๊ฐ์ ์ฑ์ (๋ณด๊ธฐ ๊ธฐํธ ์ ์, 0-shot).
# OpenAI ํธํ API๋ก ๋ ์๋ ๋ชจ๋ธ (vLLM ๋ฑ) โ ํค๋ ํ๊ฒฝ ๋ณ์ OPENAI_API_KEY ๋ก
python eval/bench_compare.py --api http://์๋ฒ:ํฌํธ/v1 [--api-model ๋ชจ๋ธ์ด๋ฆ]
# GPU ์๋ฒ์์ (๋ชจ๋ธ์ Hugging Face์์ ์๋์ผ๋ก ๋ฐ๋๋ค)
python eval/bench_compare.py --model Qwen/Qwen3-4B # ์๋ณธ ๋ชจ๋ธ
python eval/bench_compare.py --model google/gemma-3-27b-it --4bit # ํฐ ๋ชจ๋ธ์ 4bit๋ก (bitsandbytes ํ์)
python eval/bench_compare.py --model . --format jev # ์ด ๋ชจ๋ธ (ํ์ต ํ์ ๊ทธ๋๋ก)
๊ณต์ ์ฑ์ ์ํด:
- ํ๋กฌํํธ ๋ณธ๋ฌธ(๊ณผ์ ์ค๋ช
ยท ํญ๋ชฉ ยท ๋ณด๊ธฐ ยท "์ ๋ต ๋ณด๊ธฐ์ ๊ธฐํธ ํ๋๋ง ๋ตํ์ธ์.")์ ๋ชจ๋ ๋ชจ๋ธ์ ๋๊ฐ๋ค.
- ๋ํ ํ์ ๊ฐ ๋ชจ๋ธ์ chat template ์ ์ฐ๊ณ , ์๊ฐ(thinking) ๊ธฐ๋ฅ์ด ์์ผ๋ฉด ๋๋ค (Qwen3: enable_thinking=False).
- ๋ค์ ํ ํฐ์์ ๋ณด๊ธฐ ๊ธฐํธ ํ ํฐ("A" ๋๋ " A" ์ค ๋์ ์ชฝ)์ ์ ์๋ง ๋น๊ตํ๋ค. ์ ํ๋๋ ์จ๋ ๋ณด์ ๊ณผ ๋ฌด๊ดํ๋ค.
"""
import argparse
import json
import random
import sys
import time
from pathlib import Path
import torch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from bench_heldout import BENCH, load, wilson # noqa: E402 ๊ฐ์ ๋ฐ์ดํฐ ยท ๊ฐ์ ํ๋ณธ
from judge import chat_prefix, render_prompt # noqa: E402 ๊ฐ์ ํ๋กฌํํธ ๋ณธ๋ฌธ
def build_text(tok, prompt, fmt):
if fmt == "jev":
return chat_prefix(prompt)
msgs = [{"role": "user", "content": prompt}]
try:
return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False)
except TypeError:
return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
def letter_variants(tok):
out = []
for i in range(26):
L = chr(65 + i)
ids = set()
for s in (L, " " + L):
e = tok.encode(s, add_special_tokens=False)
if len(e) == 1:
ids.add(e[0])
out.append(sorted(ids))
return out
@torch.no_grad()
def run(model, tok, texts, nchoices, letters, batch):
dev = next(model.parameters()).device
enc = [tok.encode(t, add_special_tokens=False) for t in texts]
order = sorted(range(len(enc)), key=lambda i: len(enc[i]))
pred = [None] * len(enc)
pad = tok.pad_token_id if tok.pad_token_id is not None else 0
for s in range(0, len(order), batch):
grp = order[s:s + batch]
n = max(len(enc[i]) for i in grp)
x = torch.full((len(grp), n), pad, dtype=torch.long)
m = torch.zeros((len(grp), n), dtype=torch.long)
for r, i in enumerate(grp): # ์ค๋ฅธ์ชฝ ์ฑ์ฐ๊ธฐ: ๊ฐ ๋ฌธ์ฅ์ ๋ง์ง๋ง ์๋ฆฌ ์ ์๋ง ์ฝ๋๋ค
x[r, :len(enc[i])] = torch.tensor(enc[i]); m[r, :len(enc[i])] = 1
logits = model(input_ids=x.to(dev), attention_mask=m.to(dev)).logits
for r, i in enumerate(grp):
row = logits[r, len(enc[i]) - 1].float()
sc = [max(row[t].item() for t in letters[k]) for k in range(nchoices[i])]
pred[i] = max(range(len(sc)), key=sc.__getitem__)
return pred
def run_api(base, model_name, prompts, nchoices, workers=8):
"""API ๋ฐฉ์: ๊ฐ์ ํ๋กฌํํธ๋ฅผ ์ฌ์ฉ์ ๋ฉ์์ง๋ก ๋ณด๋ด๊ณ (์๊ฐ ๋), ์ฒซ ํ ํฐ ์์ ํ๋ฅ ์์ ํ์ฉํ ๋ณด๊ธฐ ๊ธฐํธ ์ค ๊ฐ์ฅ ๋์ ๊ฒ์ ๊ณ ๋ฅธ๋ค.
์์ ํ๋ฅ ์ ๋ณด๊ธฐ ๊ธฐํธ๊ฐ ํ๋๋ ์์ผ๋ฉด ์ค์ ๋ก ๋ธ ๊ธ์๋ฅผ ์ด๋ค."""
import concurrent.futures as cf
import os
import urllib.request
key = os.environ.get("OPENAI_API_KEY", "")
hdr = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})}
def one(i):
body = {"model": model_name, "messages": [{"role": "user", "content": prompts[i]}], "max_tokens": 1, "temperature": 0,
"logprobs": True, "top_logprobs": 20, "chat_template_kwargs": {"enable_thinking": False}}
for k in range(8):
try:
req = urllib.request.Request(base + "/chat/completions", json.dumps(body, ensure_ascii=False).encode(), hdr)
c = json.loads(urllib.request.urlopen(req, timeout=120).read())["choices"][0]
allowed = {chr(65 + j): j for j in range(nchoices[i])}
best = None
for t in (c.get("logprobs") or {}).get("content", [{}])[0].get("top_logprobs", []):
L = t["token"].strip()
if L in allowed and (best is None or t["logprob"] > best[1]):
best = (allowed[L], t["logprob"])
if best:
return best[0]
txt = (c["message"].get("content") or "").strip()[:1]
return allowed.get(txt, -1)
except urllib.error.HTTPError as e:
if e.code in (400, 401, 403, 404):
raise
time.sleep(2)
except Exception:
time.sleep(2)
return -1
with cf.ThreadPoolExecutor(workers) as ex:
return list(ex.map(one, range(len(prompts))))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", help="Hugging Face ๋ชจ๋ธ id ๋๋ ํด๋")
ap.add_argument("--api", help="OpenAI ํธํ API ์ฃผ์ (โฆ/v1). ํค๋ ํ๊ฒฝ ๋ณ์ OPENAI_API_KEY")
ap.add_argument("--api-model", help="API์ ๋ชจ๋ธ ์ด๋ฆ (์๋ตํ๋ฉด /models ์ ์ฒซ ๋ฒ์งธ)")
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--format", choices=["chat", "jev"], default="chat", help="jev = ์ด ๋ชจ๋ธ์ ํ์ต ํ์ (KJH-Judge ์ ์ฉ)")
ap.add_argument("--4bit", dest="q4", action="store_true", help="4bit ์์ํ๋ก ๋ถ๋ฌ์ค๊ธฐ (ํฐ ๋ชจ๋ธ์ฉ, bitsandbytes ํ์)")
ap.add_argument("--n", type=int, default=1000)
ap.add_argument("--only", nargs="*")
ap.add_argument("--batch", type=int, default=8)
a = ap.parse_args()
if not (a.model or a.api):
ap.error("--model ๋๋ --api ๊ฐ ํ์ํฉ๋๋ค")
if a.api:
import os
import urllib.request
a.api = a.api.rstrip("/")
if not a.api_model:
key = os.environ.get("OPENAI_API_KEY", "")
req = urllib.request.Request(a.api + "/models", headers={"Authorization": f"Bearer {key}"} if key else {})
a.api_model = json.loads(urllib.request.urlopen(req, timeout=30).read())["data"][0]["id"]
name = "api-" + a.api_model.replace("/", "_")
tok = model = letters = None
else:
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained(a.model)
try:
import accelerate # noqa: F401
kw = {"device_map": "auto", "dtype": torch.bfloat16}
except ImportError: # accelerate ๊ฐ ์์ผ๋ฉด ํ ์ฅ์น์ ์ง์ ์ฌ๋ฆฐ๋ค
kw = {"dtype": torch.bfloat16 if torch.cuda.is_available() else torch.float32}
if a.q4:
from transformers import BitsAndBytesConfig
kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4")
model = AutoModelForCausalLM.from_pretrained(a.model, **kw).eval()
if "device_map" not in kw:
model = model.to("cuda" if torch.cuda.is_available() else "cpu")
letters = letter_variants(tok)
name = Path(a.model).resolve().name if Path(a.model).exists() else a.model.split("/")[-1]
out = HERE / "compare" / f"{name}{'-4bit' if a.q4 else ''}{'-' + a.format if a.format == 'jev' else ''}.json"
out.parent.mkdir(exist_ok=True)
res = {}
for b in (a.only or BENCH):
t0 = time.time()
data = load(b, random.Random(f"kjh-judge:{b}"), a.n) # bench_heldout.py ์ ๊ฐ์ ์๋ โ ๊ฐ์ ๋ฌธํญ
prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r, _ in data]
nch = [len(r["choices"]) for r, _ in data]
if a.api:
pred = run_api(a.api, a.api_model, prompts, nch, a.workers)
else:
pred = run(model, tok, [build_text(tok, p, a.format) for p in prompts], nch, letters, a.batch)
k = sum(p == g for p, (_, g) in zip(pred, data))
lo, hi = wilson(k, len(data))
res[b] = {"n": len(data), "acc": round(k / len(data), 4), "ci95": [round(lo, 4), round(hi, 4)], "seconds": round(time.time() - t0)}
print(f"{b:<10} n={len(data):>5} ์ ํ๋ {k / len(data):6.1%} ({lo:.1%}~{hi:.1%}) {res[b]['seconds']}s", flush=True)
out.write_text(json.dumps({"model": a.api_model if a.api else a.model, "via": "api" if a.api else "local", "format": a.format, "4bit": a.q4, "made": time.strftime("%Y-%m-%d %H:%M"), "results": res}, ensure_ascii=False, indent=1), encoding="utf-8")
print("์ ์ฅ:", out)
if __name__ == "__main__":
main()
|