KJH-Judge-4B-ko / eval /bench_compare.py
coderecode95's picture
Upload folder using huggingface_hub
4a62dcc verified
Raw History Blame Contribute Delete
9.09 kB
"""๋‹ค๋ฅธ ๋ชจ๋ธ๊ณผ ๊ฐ™์€ ์กฐ๊ฑด์œผ๋กœ ๋น„๊ต: bench_heldout.py ์™€ ๊ฐ™์€ ๋ฐ์ดํ„ฐ ยท ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ยท ๊ฐ™์€ ์ฑ„์ (๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜, 0-shot).
# OpenAI ํ˜ธํ™˜ API๋กœ ๋–  ์žˆ๋Š” ๋ชจ๋ธ (vLLM ๋“ฑ) โ€” ํ‚ค๋Š” ํ™˜๊ฒฝ ๋ณ€์ˆ˜ OPENAI_API_KEY ๋กœ
python eval/bench_compare.py --api http://์„œ๋ฒ„:ํฌํŠธ/v1 [--api-model ๋ชจ๋ธ์ด๋ฆ„]
# GPU ์„œ๋ฒ„์—์„œ (๋ชจ๋ธ์€ Hugging Face์—์„œ ์ž๋™์œผ๋กœ ๋ฐ›๋Š”๋‹ค)
python eval/bench_compare.py --model Qwen/Qwen3-4B # ์›๋ณธ ๋ชจ๋ธ
python eval/bench_compare.py --model google/gemma-3-27b-it --4bit # ํฐ ๋ชจ๋ธ์€ 4bit๋กœ (bitsandbytes ํ•„์š”)
python eval/bench_compare.py --model . --format jev # ์ด ๋ชจ๋ธ (ํ•™์Šต ํ˜•์‹ ๊ทธ๋Œ€๋กœ)
๊ณต์ •์„ฑ์„ ์œ„ํ•ด:
- ํ”„๋กฌํ”„ํŠธ ๋ณธ๋ฌธ(๊ณผ์ œ ์„ค๋ช… ยท ํ•ญ๋ชฉ ยท ๋ณด๊ธฐ ยท "์ •๋‹ต ๋ณด๊ธฐ์˜ ๊ธฐํ˜ธ ํ•˜๋‚˜๋งŒ ๋‹ตํ•˜์„ธ์š”.")์€ ๋ชจ๋“  ๋ชจ๋ธ์— ๋˜‘๊ฐ™๋‹ค.
- ๋Œ€ํ™” ํ‹€์€ ๊ฐ ๋ชจ๋ธ์˜ chat template ์„ ์“ฐ๊ณ , ์ƒ๊ฐ(thinking) ๊ธฐ๋Šฅ์ด ์žˆ์œผ๋ฉด ๋ˆ๋‹ค (Qwen3: enable_thinking=False).
- ๋‹ค์Œ ํ† ํฐ์—์„œ ๋ณด๊ธฐ ๊ธฐํ˜ธ ํ† ํฐ("A" ๋˜๋Š” " A" ์ค‘ ๋†’์€ ์ชฝ)์˜ ์ ์ˆ˜๋งŒ ๋น„๊ตํ•œ๋‹ค. ์ •ํ™•๋„๋Š” ์˜จ๋„ ๋ณด์ •๊ณผ ๋ฌด๊ด€ํ•˜๋‹ค.
"""
import argparse
import json
import random
import sys
import time
from pathlib import Path
import torch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from bench_heldout import BENCH, load, wilson # noqa: E402 ๊ฐ™์€ ๋ฐ์ดํ„ฐ ยท ๊ฐ™์€ ํ‘œ๋ณธ
from judge import chat_prefix, render_prompt # noqa: E402 ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ๋ณธ๋ฌธ
def build_text(tok, prompt, fmt):
if fmt == "jev":
return chat_prefix(prompt)
msgs = [{"role": "user", "content": prompt}]
try:
return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False)
except TypeError:
return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
def letter_variants(tok):
out = []
for i in range(26):
L = chr(65 + i)
ids = set()
for s in (L, " " + L):
e = tok.encode(s, add_special_tokens=False)
if len(e) == 1:
ids.add(e[0])
out.append(sorted(ids))
return out
@torch.no_grad()
def run(model, tok, texts, nchoices, letters, batch):
dev = next(model.parameters()).device
enc = [tok.encode(t, add_special_tokens=False) for t in texts]
order = sorted(range(len(enc)), key=lambda i: len(enc[i]))
pred = [None] * len(enc)
pad = tok.pad_token_id if tok.pad_token_id is not None else 0
for s in range(0, len(order), batch):
grp = order[s:s + batch]
n = max(len(enc[i]) for i in grp)
x = torch.full((len(grp), n), pad, dtype=torch.long)
m = torch.zeros((len(grp), n), dtype=torch.long)
for r, i in enumerate(grp): # ์˜ค๋ฅธ์ชฝ ์ฑ„์šฐ๊ธฐ: ๊ฐ ๋ฌธ์žฅ์˜ ๋งˆ์ง€๋ง‰ ์ž๋ฆฌ ์ ์ˆ˜๋งŒ ์ฝ๋Š”๋‹ค
x[r, :len(enc[i])] = torch.tensor(enc[i]); m[r, :len(enc[i])] = 1
logits = model(input_ids=x.to(dev), attention_mask=m.to(dev)).logits
for r, i in enumerate(grp):
row = logits[r, len(enc[i]) - 1].float()
sc = [max(row[t].item() for t in letters[k]) for k in range(nchoices[i])]
pred[i] = max(range(len(sc)), key=sc.__getitem__)
return pred
def run_api(base, model_name, prompts, nchoices, workers=8):
"""API ๋ฐฉ์‹: ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ๋ฅผ ์‚ฌ์šฉ์ž ๋ฉ”์‹œ์ง€๋กœ ๋ณด๋‚ด๊ณ (์ƒ๊ฐ ๋”), ์ฒซ ํ† ํฐ ์ƒ์œ„ ํ™•๋ฅ ์—์„œ ํ—ˆ์šฉํ•œ ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ค‘ ๊ฐ€์žฅ ๋†’์€ ๊ฒƒ์„ ๊ณ ๋ฅธ๋‹ค.
์ƒ์œ„ ํ™•๋ฅ ์— ๋ณด๊ธฐ ๊ธฐํ˜ธ๊ฐ€ ํ•˜๋‚˜๋„ ์—†์œผ๋ฉด ์‹ค์ œ๋กœ ๋‚ธ ๊ธ€์ž๋ฅผ ์“ด๋‹ค."""
import concurrent.futures as cf
import os
import urllib.request
key = os.environ.get("OPENAI_API_KEY", "")
hdr = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})}
def one(i):
body = {"model": model_name, "messages": [{"role": "user", "content": prompts[i]}], "max_tokens": 1, "temperature": 0,
"logprobs": True, "top_logprobs": 20, "chat_template_kwargs": {"enable_thinking": False}}
for k in range(8):
try:
req = urllib.request.Request(base + "/chat/completions", json.dumps(body, ensure_ascii=False).encode(), hdr)
c = json.loads(urllib.request.urlopen(req, timeout=120).read())["choices"][0]
allowed = {chr(65 + j): j for j in range(nchoices[i])}
best = None
for t in (c.get("logprobs") or {}).get("content", [{}])[0].get("top_logprobs", []):
L = t["token"].strip()
if L in allowed and (best is None or t["logprob"] > best[1]):
best = (allowed[L], t["logprob"])
if best:
return best[0]
txt = (c["message"].get("content") or "").strip()[:1]
return allowed.get(txt, -1)
except urllib.error.HTTPError as e:
if e.code in (400, 401, 403, 404):
raise
time.sleep(2)
except Exception:
time.sleep(2)
return -1
with cf.ThreadPoolExecutor(workers) as ex:
return list(ex.map(one, range(len(prompts))))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", help="Hugging Face ๋ชจ๋ธ id ๋˜๋Š” ํด๋”")
ap.add_argument("--api", help="OpenAI ํ˜ธํ™˜ API ์ฃผ์†Œ (โ€ฆ/v1). ํ‚ค๋Š” ํ™˜๊ฒฝ ๋ณ€์ˆ˜ OPENAI_API_KEY")
ap.add_argument("--api-model", help="API์˜ ๋ชจ๋ธ ์ด๋ฆ„ (์ƒ๋žตํ•˜๋ฉด /models ์˜ ์ฒซ ๋ฒˆ์งธ)")
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--format", choices=["chat", "jev"], default="chat", help="jev = ์ด ๋ชจ๋ธ์˜ ํ•™์Šต ํ˜•์‹ (KJH-Judge ์ „์šฉ)")
ap.add_argument("--4bit", dest="q4", action="store_true", help="4bit ์–‘์žํ™”๋กœ ๋ถˆ๋Ÿฌ์˜ค๊ธฐ (ํฐ ๋ชจ๋ธ์šฉ, bitsandbytes ํ•„์š”)")
ap.add_argument("--n", type=int, default=1000)
ap.add_argument("--only", nargs="*")
ap.add_argument("--batch", type=int, default=8)
a = ap.parse_args()
if not (a.model or a.api):
ap.error("--model ๋˜๋Š” --api ๊ฐ€ ํ•„์š”ํ•ฉ๋‹ˆ๋‹ค")
if a.api:
import os
import urllib.request
a.api = a.api.rstrip("/")
if not a.api_model:
key = os.environ.get("OPENAI_API_KEY", "")
req = urllib.request.Request(a.api + "/models", headers={"Authorization": f"Bearer {key}"} if key else {})
a.api_model = json.loads(urllib.request.urlopen(req, timeout=30).read())["data"][0]["id"]
name = "api-" + a.api_model.replace("/", "_")
tok = model = letters = None
else:
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained(a.model)
try:
import accelerate # noqa: F401
kw = {"device_map": "auto", "dtype": torch.bfloat16}
except ImportError: # accelerate ๊ฐ€ ์—†์œผ๋ฉด ํ•œ ์žฅ์น˜์— ์ง์ ‘ ์˜ฌ๋ฆฐ๋‹ค
kw = {"dtype": torch.bfloat16 if torch.cuda.is_available() else torch.float32}
if a.q4:
from transformers import BitsAndBytesConfig
kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4")
model = AutoModelForCausalLM.from_pretrained(a.model, **kw).eval()
if "device_map" not in kw:
model = model.to("cuda" if torch.cuda.is_available() else "cpu")
letters = letter_variants(tok)
name = Path(a.model).resolve().name if Path(a.model).exists() else a.model.split("/")[-1]
out = HERE / "compare" / f"{name}{'-4bit' if a.q4 else ''}{'-' + a.format if a.format == 'jev' else ''}.json"
out.parent.mkdir(exist_ok=True)
res = {}
for b in (a.only or BENCH):
t0 = time.time()
data = load(b, random.Random(f"kjh-judge:{b}"), a.n) # bench_heldout.py ์™€ ๊ฐ™์€ ์‹œ๋“œ โ†’ ๊ฐ™์€ ๋ฌธํ•ญ
prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r, _ in data]
nch = [len(r["choices"]) for r, _ in data]
if a.api:
pred = run_api(a.api, a.api_model, prompts, nch, a.workers)
else:
pred = run(model, tok, [build_text(tok, p, a.format) for p in prompts], nch, letters, a.batch)
k = sum(p == g for p, (_, g) in zip(pred, data))
lo, hi = wilson(k, len(data))
res[b] = {"n": len(data), "acc": round(k / len(data), 4), "ci95": [round(lo, 4), round(hi, 4)], "seconds": round(time.time() - t0)}
print(f"{b:<10} n={len(data):>5} ์ •ํ™•๋„ {k / len(data):6.1%} ({lo:.1%}~{hi:.1%}) {res[b]['seconds']}s", flush=True)
out.write_text(json.dumps({"model": a.api_model if a.api else a.model, "via": "api" if a.api else "local", "format": a.format, "4bit": a.q4, "made": time.strftime("%Y-%m-%d %H:%M"), "results": res}, ensure_ascii=False, indent=1), encoding="utf-8")
print("์ €์žฅ:", out)
if __name__ == "__main__":
main()