KJH-Judge-4B-ko / eval /bench_heldout.py
coderecode95's picture
Upload folder using huggingface_hub
4a62dcc verified
Raw History Blame Contribute Delete
10.1 kB
"""ํ•™์Šต์— ์“ฐ์ง€ ์•Š์€ ๊ณต๊ฐœ ํ•œ๊ตญ์–ด ๋ฒค์น˜๋งˆํฌ๋กœ KJH-Judge-4B-ko ๋ฅผ ์žฐ๋‹ค (0-shot, ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜ ๋ฐฉ์‹).
# ํŒ์ • ์„œ๋ฒ„(API)๋กœ
python eval/bench_heldout.py --url http://์„œ๋ฒ„:ํฌํŠธ
# ๋‚ด๋ ค๋ฐ›์€ ๋ชจ๋ธ๋กœ (GPU ๊ถŒ์žฅ)
python eval/bench_heldout.py --model .
ํ•™์Šต ๋ฐ์ดํ„ฐ์™€ ๊ฒน์น˜์ง€ ์•Š๋Š” ์ถœ์ฒ˜๋งŒ ์“ด๋‹ค. ์ด ๋ชจ๋ธ์˜ ํ•™์Šต ๋ฐ์ดํ„ฐ๋Š” KLUE(NLIยทYNAT), KoBEST, korean-hate-speech, APEACH, SIB-200,
์ž์ฒด ํ•ฉ์„ฑ(์ •์ฑ…ยท๋ฒ•๋ นยท๋„๊ตฌ ํ˜ธ์ถœ)์ด๋‹ค. ์•„๋ž˜ ๋ฒค์น˜๋งˆํฌ๋Š” ๊ทธ์™€ ๋‹ค๋ฅธ ์ถœ์ฒ˜๋‹ค. (์›๋ณธ Qwen3-4B์˜ ์‚ฌ์ „ํ•™์Šต์— ๋“ค์–ด๊ฐ”๋Š”์ง€๋Š” ์•Œ ์ˆ˜ ์—†๋‹ค.)
ํ‘œ๋ณธ์€ ๊ณ ์ • ์‹œ๋“œ๋กœ ๋ฝ‘๋Š”๋‹ค.
"""
import argparse
import concurrent.futures as cf
import io
import json
import math
import random
import time
import urllib.parse
import urllib.request
from pathlib import Path
HERE = Path(__file__).resolve().parent
NLI = ("์ „์ œ๋ฅผ ์ฐธ์ด๋ผ๊ณ  ํ•  ๋•Œ ๊ฐ€์„ค์„ ๋ถ„๋ฅ˜ํ•œ๋‹ค. entailment: ์ „์ œ๋กœ๋ถ€ํ„ฐ ๊ฐ€์„ค์ด ๋”ฐ๋ผ์˜จ๋‹ค. contradiction: ์ „์ œ์™€ ๊ฐ€์„ค์ด ์–‘๋ฆฝํ•  ์ˆ˜ ์—†๋‹ค. neutral: ๋‘˜ ๋‹ค ์•„๋‹ˆ๋‹ค. "
"์ •๋ณด๊ฐ€ ์—†๋‹ค๋Š” ์ด์œ ๋งŒ์œผ๋กœ contradiction์„ ๊ณ ๋ฅด์ง€ ์•Š๋Š”๋‹ค. neutral์€ ์ •์ƒ ์ •๋‹ต์ด๋ฉฐ insufficient_context๊ฐ€ ์•„๋‹ˆ๋‹ค. ํ†ต์ƒ์ ์ธ ๋ฌธ์žฅ ์˜๋ฏธ๋กœ ํŒ๋‹จํ•˜๋ฉฐ ํฌ๋ฐ•ํ•œ ๊ฐ€์ •์„ ๋งŒ๋“ค์–ด ๋ชจํ˜ธํ•˜๋‹ค๊ณ  ํ•˜์ง€ ์•Š๋Š”๋‹ค. "
"์ „์ œ์˜ ์—ด๊ฑฐ ๋Œ€์ƒ๊ณผ ๊ฐ€์„ค์˜ ๋ฒ”์œ„๋ฅผ ๋น ์ง์—†์ด ๋Œ€์กฐํ•œ๋‹ค. ๊ฐ€์„ค์˜ ๋งŒ/๋ฐ–์—/์˜ค์ง/๋ชจ๋‘ ๊ฐ™์€ ํ‘œํ˜„์ด ์ „์ œ์— ๋ช…์‹œ๋œ ๋‹ค๋ฅธ ๋Œ€์ƒ์ด๋‚˜ ์‚ฌ๋ก€๋ฅผ ๋ฐฐ์ œํ•˜๋Š”์ง€ ํ™•์ธํ•œ๋‹ค. "
"์ „์ œ์— ๋ฐฐํƒ€ ํ‘œํ˜„์ด ์—†๋‹ค๋Š” ์‚ฌ์‹ค๋งŒ์œผ๋กœ ์ค‘๋ฆฝ์ด๋ผ๊ณ  ํ•˜์ง€ ์•Š๋Š”๋‹ค. ํ†ต์ƒ์ ์ธ ๋ฌธ๋งฅ์ด ๋ช…ํ™•ํ•˜๋ฉด ๋‹ค๋ฅธ ์›์ธยท์ƒํ™ฉ์„ ์ž„์˜๋กœ ์ถ”๊ฐ€ํ•˜์ง€ ์•Š๋Š”๋‹ค.")
SENT = "๋ฌธ์žฅ์˜ ์ „๋ฐ˜์ ์ธ ๊ฐ์ • ๊ทน์„ฑ์„ ํŒ๋‹จํ•œ๋‹ค. ๋ถ€์ • ํ‘œํ˜„๊ณผ ๋ฐ˜์–ด๋ฅผ ๊ณ ๋ คํ•œ๋‹ค."
HATE = ("APEACH ์› ๋…ผ๋ฌธ์˜ ์ƒ์„ฑ ๊ธฐ์ค€์„ ๋”ฐ๋ฅธ๋‹ค. ์„ฑ๋ณ„ยท์„ฑ์  ์ง€ํ–ฅ/์ •์ฒด์„ฑยท๋‚˜์ด/์‹ ๋ถ„/๊ฒฝ๋ ฅยท๊ตญ์ /์ธ์ข…ยท์ถœ์‹ /๊ฑฐ์ฃผ์ง€ยท์™ธ๋ชจยท์งˆ๋ณ‘/์žฅ์• ยท์‹์Šต๊ด€ยทํ•™๋ ฅ/์ „๊ณต ๊ด€๋ จ ์ฐจ๋ณ„ยท๋น„ํ•˜ยท๊ณต๊ฒฉ์„ ํฌํ•จํ•œ๋‹ค. "
"๋ณดํ˜ธ์ง‘๋‹จ ์ •์ฒด์„ฑ์„ ๋ช…์‹œํ•ด์•ผ ํ•œ๋‹ค๋Š” ์ œํ•œ์€ ์—†๋‹ค. ๋ถˆํ•„์š”ํ•˜๊ฑฐ๋‚˜ ๊ณต๊ฒฉ์ ์ธ ์™ธ๋ชจ ํ‰๊ฐ€, ์‹์Šต๊ด€ ๊ฐ•์š”/ํ„ํ•˜๋„ ํ•ด๋‹นํ•œ๋‹ค. ๋น„๊ณต๊ฒฉ์ ์ธ ์ฃผ์ œ ์–ธ๊ธ‰์ด๋‚˜ ์ฐจ๋ณ„์— ๋Œ€ํ•œ ๋ฐ˜๋Œ€ ๋ฐœ์–ธ์€ ๋น„ํ˜์˜ค๋‹ค.")
STS = "๋‘ ๋ฌธ์žฅ์˜ ์˜๋ฏธ๊ฐ€ ๊ฐ™์€์ง€ ํŒ๋‹จํ•œ๋‹ค. ํ‘œํ˜„์ด ๋‹ฌ๋ผ๋„ ์ „๋‹ฌํ•˜๋Š” ๋‚ด์šฉ์ด ๊ฐ™์œผ๋ฉด ๊ฐ™์€ ์˜๋ฏธ๋‹ค."
MCQ = "์งˆ๋ฌธ์— ๊ฐ€์žฅ ์•Œ๋งž์€ ๋‹ต์„ ๋ณด๊ธฐ์—์„œ ๊ณ ๋ฅธ๋‹ค."
BENCH = { # ์ด๋ฆ„: (์„ค๋ช…, ์ถœ์ฒ˜, ๋ผ์ด์„ ์Šค)
"nsmc": ("์˜ํ™” ๋ฆฌ๋ทฐ ๊ฐ์„ฑ (๊ธ์ •/๋ถ€์ •)", "e9t/nsmc ยท ratings_test", "CC0 1.0"),
"kornli": ("๋ฌธ์žฅ ๊ด€๊ณ„ 3๋ถ„๋ฅ˜ (XNLI ํ•œ๊ตญ์–ด ๋ฒˆ์—ญ)", "kakaobrain/kor_nli ยท xnli/test", "CC BY-SA 4.0"),
"klue_sts": ("๋‘ ๋ฌธ์žฅ์ด ๊ฐ™์€ ๋œป์ธ์ง€ (์˜ˆ/์•„๋‹ˆ์˜ค)", "klue/klue ยท sts/validation (binary-label)", "CC BY-SA 4.0"),
"unsmile": ("์•…ํ”Œยทํ˜์˜ค ํ‘œํ˜„ ์—ฌ๋ถ€", "smilegate-ai/kor_unsmile ยท valid (clean ์—ฌ๋ถ€)", "CC BY-NC-ND 4.0"),
"kmmlu": ("ํ•œ๊ตญ์–ด ์ „๋ฌธ ์ง€์‹ 4์ง€์„ ๋‹ค (45๊ณผ๋ชฉ)", "HAERAE-HUB/KMMLU ยท test", "CC BY-ND 4.0"),
"haerae": ("ํ•œ๊ตญ ๋ฌธํ™”ยท์–ดํœ˜ ์ง€์‹ 5์ง€์„ ๋‹ค", "HAERAE-HUB/HAE_RAE_BENCH_1.0 ยท test", "CC BY-NC-ND 4.0"),
}
def get(url, tries=6):
url = urllib.parse.quote(url, safe=":/?=&%") # ์„ค์ • ์ด๋ฆ„์— ๋„์–ด์“ฐ๊ธฐ๊ฐ€ ์žˆ๋Š” ๋ฐ์ดํ„ฐ์…‹ (HAE-RAE)
for k in range(tries):
try:
return urllib.request.urlopen(url, timeout=120).read()
except Exception:
if k == tries - 1:
raise
time.sleep(3)
def parquet(dataset, config, split):
import pyarrow.parquet as pq
urls = json.loads(get(f"https://huggingface.co/api/datasets/{dataset}/parquet"))[config][split]
rows = []
for u in urls:
rows += pq.read_table(io.BytesIO(get(u))).to_pylist()
return rows
def load(name, rng, n):
"""โ†’ [(์š”์ฒญ, ์ •๋‹ต ๋ฒˆํ˜ธ)]"""
out = []
if name == "nsmc":
lines = get("https://raw.githubusercontent.com/e9t/nsmc/master/ratings_test.txt").decode("utf-8").splitlines()[1:]
rows = [l.split("\t") for l in lines]
rows = [r for r in rows if len(r) == 3 and r[1].strip()]
for _, doc, lab in rng.sample(rows, n):
out.append(({"instruction": SENT, "item": {"text": doc}, "choices": ["๊ธ์ •์  ๊ฐ์ •", "๋ถ€์ •์  ๊ฐ์ •"]}, 0 if lab == "1" else 1))
elif name == "kornli":
rows = parquet("kakaobrain/kor_nli", "xnli", "test")
gmap = {0: 0, 1: 2, 2: 1} # ๋ฐ์ดํ„ฐ: 0 ํ•จ์˜ 1 ์ค‘๋ฆฝ 2 ๋ชจ์ˆœ โ†’ ๋ณด๊ธฐ: entailment contradiction neutral
for r in rng.sample(rows, n):
out.append(({"instruction": NLI, "item": {"premise": r["premise"], "hypothesis": r["hypothesis"]}, "choices": ["entailment", "contradiction", "neutral"]}, gmap[r["label"]]))
elif name == "klue_sts":
for r in parquet("klue/klue", "sts", "validation"):
out.append(({"instruction": STS, "item": {"sentence1": r["sentence1"], "sentence2": r["sentence2"]}, "choices": ["๊ฐ™์€ ์˜๋ฏธ", "๋‹ค๋ฅธ ์˜๋ฏธ"]}, 0 if r["labels"]["binary-label"] == 1 else 1))
elif name == "unsmile":
for r in rng.sample(parquet("smilegate-ai/kor_unsmile", "default", "valid"), n):
out.append(({"instruction": HATE, "item": {"text": r["๋ฌธ์žฅ"]}, "choices": ["๋น„ํ˜์˜ค: ๋น„ํ•˜ยท๊ณต๊ฒฉ์„ฑ์ด ์—†๋Š” ํ‘œํ˜„", "ํ˜์˜คยท๋น„ํ•˜ยท๊ณต๊ฒฉ์  ํ‘œํ˜„"]}, 0 if r["clean"] == 1 else 1))
elif name == "kmmlu":
cfgs = json.loads(get("https://huggingface.co/api/datasets/HAERAE-HUB/KMMLU/parquet"))
per = max(1, n // len(cfgs))
for c in sorted(cfgs):
rows = parquet("HAERAE-HUB/KMMLU", c, "test")
for r in rng.sample(rows, min(per, len(rows))):
out.append(({"instruction": MCQ, "item": {"question": r["question"]}, "choices": [str(r[k]) for k in "ABCD"]}, int(r["answer"]) - 1))
elif name == "haerae":
cfgs = json.loads(get("https://huggingface.co/api/datasets/HAERAE-HUB/HAE_RAE_BENCH_1.0/parquet"))
per = max(1, n // len(cfgs))
for c in sorted(cfgs):
rows = parquet("HAERAE-HUB/HAE_RAE_BENCH_1.0", c, "test")
for r in rng.sample(rows, min(per, len(rows))):
ks = [k for k in "abcde" if r.get(k) not in (None, "")]
gold = str(r["answer"]).strip().lower()
if gold not in ks:
continue
out.append(({"instruction": MCQ, "item": {"question": r["question"]}, "choices": [str(r[k]) for k in ks]}, ks.index(gold)))
return out
def wilson(k, n, z=1.96):
if n == 0:
return (0, 0)
p = k / n
d = 1 + z * z / n
c = (p + z * z / (2 * n)) / d
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
return (c - h, c + h)
def run_api(url, reqs):
def post(path, body):
for k in range(12):
try:
req = urllib.request.Request(url + path, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"})
return json.loads(urllib.request.urlopen(req, timeout=300).read())
except urllib.error.HTTPError as e:
if e.code == 404:
return None
raise
except Exception:
time.sleep(2)
raise RuntimeError("์„œ๋ฒ„์— ์—ฐ๊ฒฐํ•˜์ง€ ๋ชปํ–ˆ์Šต๋‹ˆ๋‹ค")
wire = [{"instruction": r["instruction"], "item": r["item"], "choices": [{"text": c} for c in r["choices"]]} for r in reqs]
first = post("/api/batch", {"items": wire[:32]})
if first:
res = first["results"]
for s in range(32, len(wire), 32):
res += post("/api/batch", {"items": wire[s:s + 32]})["results"]
else:
with cf.ThreadPoolExecutor(8) as ex:
res = list(ex.map(lambda b: post("/api/decide", b), wire))
return [ord(r["choice"]) - 65 for r in res], [r["confidence"] for r in res]
def run_local(path, reqs):
import sys
sys.path.insert(0, str(Path(path).resolve()))
from judge import Judge
j = Judge(path)
out = j.decide_batch(reqs)
return [o["index"] for o in out], [o["confidence"] for o in out]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--url")
ap.add_argument("--model")
ap.add_argument("--n", type=int, default=1000, help="๋ฒค์น˜๋งˆํฌ๋งˆ๋‹ค ์ตœ๋Œ€ ๋ฌธํ•ญ ์ˆ˜ (KLUE-STS๋Š” ์ „์ฒด 519)")
ap.add_argument("--only", nargs="*")
ap.add_argument("--out", default=str(HERE / "heldout_results.json"))
a = ap.parse_args()
prev = Path(a.out)
summary = json.loads(prev.read_text(encoding="utf-8"))["results"] if prev.is_file() and a.only else {} # --only ๋กœ ์ผ๋ถ€๋งŒ ๋‹ค์‹œ ๋Œ๋ฆฌ๋ฉด ๋‚˜๋จธ์ง€ ๊ฒฐ๊ณผ๋Š” ์œ ์ง€
for name in (a.only or BENCH):
rng = random.Random(f"kjh-judge:{name}")
t0 = time.time()
data = load(name, rng, a.n)
reqs, gold = [d[0] for d in data], [d[1] for d in data]
pred, conf = run_api(a.url.rstrip("/"), reqs) if a.url else run_local(a.model, reqs)
k = sum(p == g for p, g in zip(pred, gold))
chance = sum(1 / len(r["choices"]) for r in reqs) / len(reqs)
lo, hi = wilson(k, len(reqs))
summary[name] = {"desc": BENCH[name][0], "source": BENCH[name][1], "license": BENCH[name][2], "n": len(reqs),
"acc": round(k / len(reqs), 4), "ci95": [round(lo, 4), round(hi, 4)], "chance": round(chance, 4),
"mean_conf": round(sum(conf) / len(conf), 4), "seconds": round(time.time() - t0)}
print(f"{name:<10} n={len(reqs):>5} ์ •ํ™•๋„ {k / len(reqs):6.1%} (95% {lo:.1%}~{hi:.1%}) ๋ฌด์ž‘์œ„ {chance:.1%} {round(time.time() - t0)}s", flush=True)
Path(a.out).write_text(json.dumps({"made": time.strftime("%Y-%m-%d %H:%M"), "protocol": "0-shot, ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜(์˜จ๋„ ๋ณด์ • softmax), ํ‘œ๋ณธ ๊ณ ์ • ์‹œ๋“œ", "results": summary}, ensure_ascii=False, indent=1), encoding="utf-8")
print("์ €์žฅ:", a.out)
if __name__ == "__main__":
main()