Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download eval/bench_compare.py from coderecode95/KJH-Judge-4B-ko: direct link, hf CLI and curl.
- Browser
- Download file 9.09 kB
-
https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/bench_compare.py
- Command line
-
hf download hf://coderecode95/KJH-Judge-4B-ko/eval/bench_compare.py
-
curl -L -o bench_compare.py https://huggingface.co/coderecode95/KJH-Judge-4B-ko/resolve/main/eval/bench_compare.py
9.09 kB
| """๋ค๋ฅธ ๋ชจ๋ธ๊ณผ ๊ฐ์ ์กฐ๊ฑด์ผ๋ก ๋น๊ต: bench_heldout.py ์ ๊ฐ์ ๋ฐ์ดํฐ ยท ๊ฐ์ ํ๋กฌํํธ ยท ๊ฐ์ ์ฑ์ (๋ณด๊ธฐ ๊ธฐํธ ์ ์, 0-shot). | |
| # OpenAI ํธํ API๋ก ๋ ์๋ ๋ชจ๋ธ (vLLM ๋ฑ) โ ํค๋ ํ๊ฒฝ ๋ณ์ OPENAI_API_KEY ๋ก | |
| python eval/bench_compare.py --api http://์๋ฒ:ํฌํธ/v1 [--api-model ๋ชจ๋ธ์ด๋ฆ] | |
| # GPU ์๋ฒ์์ (๋ชจ๋ธ์ Hugging Face์์ ์๋์ผ๋ก ๋ฐ๋๋ค) | |
| python eval/bench_compare.py --model Qwen/Qwen3-4B # ์๋ณธ ๋ชจ๋ธ | |
| python eval/bench_compare.py --model google/gemma-3-27b-it --4bit # ํฐ ๋ชจ๋ธ์ 4bit๋ก (bitsandbytes ํ์) | |
| python eval/bench_compare.py --model . --format jev # ์ด ๋ชจ๋ธ (ํ์ต ํ์ ๊ทธ๋๋ก) | |
| ๊ณต์ ์ฑ์ ์ํด: | |
| - ํ๋กฌํํธ ๋ณธ๋ฌธ(๊ณผ์ ์ค๋ช ยท ํญ๋ชฉ ยท ๋ณด๊ธฐ ยท "์ ๋ต ๋ณด๊ธฐ์ ๊ธฐํธ ํ๋๋ง ๋ตํ์ธ์.")์ ๋ชจ๋ ๋ชจ๋ธ์ ๋๊ฐ๋ค. | |
| - ๋ํ ํ์ ๊ฐ ๋ชจ๋ธ์ chat template ์ ์ฐ๊ณ , ์๊ฐ(thinking) ๊ธฐ๋ฅ์ด ์์ผ๋ฉด ๋๋ค (Qwen3: enable_thinking=False). | |
| - ๋ค์ ํ ํฐ์์ ๋ณด๊ธฐ ๊ธฐํธ ํ ํฐ("A" ๋๋ " A" ์ค ๋์ ์ชฝ)์ ์ ์๋ง ๋น๊ตํ๋ค. ์ ํ๋๋ ์จ๋ ๋ณด์ ๊ณผ ๋ฌด๊ดํ๋ค. | |
| """ | |
| import argparse | |
| import json | |
| import random | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import torch | |
| HERE = Path(__file__).resolve().parent | |
| sys.path.insert(0, str(HERE)) | |
| sys.path.insert(0, str(HERE.parent)) | |
| from bench_heldout import BENCH, load, wilson # noqa: E402 ๊ฐ์ ๋ฐ์ดํฐ ยท ๊ฐ์ ํ๋ณธ | |
| from judge import chat_prefix, render_prompt # noqa: E402 ๊ฐ์ ํ๋กฌํํธ ๋ณธ๋ฌธ | |
| def build_text(tok, prompt, fmt): | |
| if fmt == "jev": | |
| return chat_prefix(prompt) | |
| msgs = [{"role": "user", "content": prompt}] | |
| try: | |
| return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False) | |
| except TypeError: | |
| return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) | |
| def letter_variants(tok): | |
| out = [] | |
| for i in range(26): | |
| L = chr(65 + i) | |
| ids = set() | |
| for s in (L, " " + L): | |
| e = tok.encode(s, add_special_tokens=False) | |
| if len(e) == 1: | |
| ids.add(e[0]) | |
| out.append(sorted(ids)) | |
| return out | |
| def run(model, tok, texts, nchoices, letters, batch): | |
| dev = next(model.parameters()).device | |
| enc = [tok.encode(t, add_special_tokens=False) for t in texts] | |
| order = sorted(range(len(enc)), key=lambda i: len(enc[i])) | |
| pred = [None] * len(enc) | |
| pad = tok.pad_token_id if tok.pad_token_id is not None else 0 | |
| for s in range(0, len(order), batch): | |
| grp = order[s:s + batch] | |
| n = max(len(enc[i]) for i in grp) | |
| x = torch.full((len(grp), n), pad, dtype=torch.long) | |
| m = torch.zeros((len(grp), n), dtype=torch.long) | |
| for r, i in enumerate(grp): # ์ค๋ฅธ์ชฝ ์ฑ์ฐ๊ธฐ: ๊ฐ ๋ฌธ์ฅ์ ๋ง์ง๋ง ์๋ฆฌ ์ ์๋ง ์ฝ๋๋ค | |
| x[r, :len(enc[i])] = torch.tensor(enc[i]); m[r, :len(enc[i])] = 1 | |
| logits = model(input_ids=x.to(dev), attention_mask=m.to(dev)).logits | |
| for r, i in enumerate(grp): | |
| row = logits[r, len(enc[i]) - 1].float() | |
| sc = [max(row[t].item() for t in letters[k]) for k in range(nchoices[i])] | |
| pred[i] = max(range(len(sc)), key=sc.__getitem__) | |
| return pred | |
| def run_api(base, model_name, prompts, nchoices, workers=8): | |
| """API ๋ฐฉ์: ๊ฐ์ ํ๋กฌํํธ๋ฅผ ์ฌ์ฉ์ ๋ฉ์์ง๋ก ๋ณด๋ด๊ณ (์๊ฐ ๋), ์ฒซ ํ ํฐ ์์ ํ๋ฅ ์์ ํ์ฉํ ๋ณด๊ธฐ ๊ธฐํธ ์ค ๊ฐ์ฅ ๋์ ๊ฒ์ ๊ณ ๋ฅธ๋ค. | |
| ์์ ํ๋ฅ ์ ๋ณด๊ธฐ ๊ธฐํธ๊ฐ ํ๋๋ ์์ผ๋ฉด ์ค์ ๋ก ๋ธ ๊ธ์๋ฅผ ์ด๋ค.""" | |
| import concurrent.futures as cf | |
| import os | |
| import urllib.request | |
| key = os.environ.get("OPENAI_API_KEY", "") | |
| hdr = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})} | |
| def one(i): | |
| body = {"model": model_name, "messages": [{"role": "user", "content": prompts[i]}], "max_tokens": 1, "temperature": 0, | |
| "logprobs": True, "top_logprobs": 20, "chat_template_kwargs": {"enable_thinking": False}} | |
| for k in range(8): | |
| try: | |
| req = urllib.request.Request(base + "/chat/completions", json.dumps(body, ensure_ascii=False).encode(), hdr) | |
| c = json.loads(urllib.request.urlopen(req, timeout=120).read())["choices"][0] | |
| allowed = {chr(65 + j): j for j in range(nchoices[i])} | |
| best = None | |
| for t in (c.get("logprobs") or {}).get("content", [{}])[0].get("top_logprobs", []): | |
| L = t["token"].strip() | |
| if L in allowed and (best is None or t["logprob"] > best[1]): | |
| best = (allowed[L], t["logprob"]) | |
| if best: | |
| return best[0] | |
| txt = (c["message"].get("content") or "").strip()[:1] | |
| return allowed.get(txt, -1) | |
| except urllib.error.HTTPError as e: | |
| if e.code in (400, 401, 403, 404): | |
| raise | |
| time.sleep(2) | |
| except Exception: | |
| time.sleep(2) | |
| return -1 | |
| with cf.ThreadPoolExecutor(workers) as ex: | |
| return list(ex.map(one, range(len(prompts)))) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", help="Hugging Face ๋ชจ๋ธ id ๋๋ ํด๋") | |
| ap.add_argument("--api", help="OpenAI ํธํ API ์ฃผ์ (โฆ/v1). ํค๋ ํ๊ฒฝ ๋ณ์ OPENAI_API_KEY") | |
| ap.add_argument("--api-model", help="API์ ๋ชจ๋ธ ์ด๋ฆ (์๋ตํ๋ฉด /models ์ ์ฒซ ๋ฒ์งธ)") | |
| ap.add_argument("--workers", type=int, default=8) | |
| ap.add_argument("--format", choices=["chat", "jev"], default="chat", help="jev = ์ด ๋ชจ๋ธ์ ํ์ต ํ์ (KJH-Judge ์ ์ฉ)") | |
| ap.add_argument("--4bit", dest="q4", action="store_true", help="4bit ์์ํ๋ก ๋ถ๋ฌ์ค๊ธฐ (ํฐ ๋ชจ๋ธ์ฉ, bitsandbytes ํ์)") | |
| ap.add_argument("--n", type=int, default=1000) | |
| ap.add_argument("--only", nargs="*") | |
| ap.add_argument("--batch", type=int, default=8) | |
| a = ap.parse_args() | |
| if not (a.model or a.api): | |
| ap.error("--model ๋๋ --api ๊ฐ ํ์ํฉ๋๋ค") | |
| if a.api: | |
| import os | |
| import urllib.request | |
| a.api = a.api.rstrip("/") | |
| if not a.api_model: | |
| key = os.environ.get("OPENAI_API_KEY", "") | |
| req = urllib.request.Request(a.api + "/models", headers={"Authorization": f"Bearer {key}"} if key else {}) | |
| a.api_model = json.loads(urllib.request.urlopen(req, timeout=30).read())["data"][0]["id"] | |
| name = "api-" + a.api_model.replace("/", "_") | |
| tok = model = letters = None | |
| else: | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| tok = AutoTokenizer.from_pretrained(a.model) | |
| try: | |
| import accelerate # noqa: F401 | |
| kw = {"device_map": "auto", "dtype": torch.bfloat16} | |
| except ImportError: # accelerate ๊ฐ ์์ผ๋ฉด ํ ์ฅ์น์ ์ง์ ์ฌ๋ฆฐ๋ค | |
| kw = {"dtype": torch.bfloat16 if torch.cuda.is_available() else torch.float32} | |
| if a.q4: | |
| from transformers import BitsAndBytesConfig | |
| kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4") | |
| model = AutoModelForCausalLM.from_pretrained(a.model, **kw).eval() | |
| if "device_map" not in kw: | |
| model = model.to("cuda" if torch.cuda.is_available() else "cpu") | |
| letters = letter_variants(tok) | |
| name = Path(a.model).resolve().name if Path(a.model).exists() else a.model.split("/")[-1] | |
| out = HERE / "compare" / f"{name}{'-4bit' if a.q4 else ''}{'-' + a.format if a.format == 'jev' else ''}.json" | |
| out.parent.mkdir(exist_ok=True) | |
| res = {} | |
| for b in (a.only or BENCH): | |
| t0 = time.time() | |
| data = load(b, random.Random(f"kjh-judge:{b}"), a.n) # bench_heldout.py ์ ๊ฐ์ ์๋ โ ๊ฐ์ ๋ฌธํญ | |
| prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r, _ in data] | |
| nch = [len(r["choices"]) for r, _ in data] | |
| if a.api: | |
| pred = run_api(a.api, a.api_model, prompts, nch, a.workers) | |
| else: | |
| pred = run(model, tok, [build_text(tok, p, a.format) for p in prompts], nch, letters, a.batch) | |
| k = sum(p == g for p, (_, g) in zip(pred, data)) | |
| lo, hi = wilson(k, len(data)) | |
| res[b] = {"n": len(data), "acc": round(k / len(data), 4), "ci95": [round(lo, 4), round(hi, 4)], "seconds": round(time.time() - t0)} | |
| print(f"{b:<10} n={len(data):>5} ์ ํ๋ {k / len(data):6.1%} ({lo:.1%}~{hi:.1%}) {res[b]['seconds']}s", flush=True) | |
| out.write_text(json.dumps({"model": a.api_model if a.api else a.model, "via": "api" if a.api else "local", "format": a.format, "4bit": a.q4, "made": time.strftime("%Y-%m-%d %H:%M"), "results": res}, ensure_ascii=False, indent=1), encoding="utf-8") | |
| print("์ ์ฅ:", out) | |
| if __name__ == "__main__": | |
| main() | |