KJH-Judge-4B-ko / eval /fill_card.py
coderecode95's picture
Upload folder using huggingface_hub
4a62dcc verified
Raw History Blame Contribute Delete
4.08 kB
"""eval/heldout_results.json ์œผ๋กœ README.md ๋ฅผ ์ฑ„์šด๋‹ค.
- ๋ณธ๋ฌธ์˜ 'ํ•™์Šตํ•˜์ง€ ์•Š์€ ๊ณต๊ฐœ ๋ฒค์น˜๋งˆํฌ' ํ‘œ
- ๋งจ ์œ„ YAML ์˜ model-index (๋ชจ๋ธ ํŽ˜์ด์ง€ ์˜ค๋ฅธ์ชฝ 'Evaluation results' ์ƒ์ž)
python eval/fill_card.py [--repo ๊ณ„์ •/KJH-Judge-4B-ko]
"""
import argparse
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
NAMES = {"nsmc": "NSMC", "kornli": "KorNLI (XNLI)", "klue_sts": "KLUE-STS", "unsmile": "Korean UnSmile", "kmmlu": "KMMLU", "haerae": "HAE-RAE Bench 1.0"}
META = { # model-index: (๊ณผ์ œ ์ข…๋ฅ˜, ๊ณผ์ œ ์ด๋ฆ„, ๋ฐ์ดํ„ฐ์…‹ id, ์„ค์ •, ๋ถ„ํ• )
"nsmc": ("text-classification", "Sentiment classification", "e9t/nsmc", None, "test"),
"kornli": ("text-classification", "Natural language inference", "kakaobrain/kor_nli", "xnli", "test"),
"klue_sts": ("text-classification", "Paraphrase detection (binary STS)", "klue/klue", "sts", "validation"),
"unsmile": ("text-classification", "Hate speech detection", "smilegate-ai/kor_unsmile", None, "valid"),
"kmmlu": ("multiple-choice", "Knowledge QA", "HAERAE-HUB/KMMLU", None, "test"),
"haerae": ("multiple-choice", "Korean knowledge QA", "HAERAE-HUB/HAE_RAE_BENCH_1.0", None, "test"),
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--repo", help="๊ณ„์ •/์ €์žฅ์†Œ ์ด๋ฆ„. ์ฃผ๋ฉด ์นด๋“œ ์•ˆ์˜ KJH/KJH-Judge-4B-ko ๋ฅผ ๋ฐ”๊พผ๋‹ค")
a = ap.parse_args()
R = json.loads((ROOT / "eval" / "heldout_results.json").read_text(encoding="utf-8"))["results"]
card = (ROOT / "README.md").read_text(encoding="utf-8")
rows = ["| ๋ฒค์น˜๋งˆํฌ | ๋ฌด์—‡์„ ์žฌ๋‚˜ | ๋ฌธํ•ญ | ์ •ํ™•๋„ (95% ๊ตฌ๊ฐ„) | ๋ฌด์ž‘์œ„ | ์ถœ์ฒ˜ ยท ๋ผ์ด์„ ์Šค |", "|---|---|---:|---:|---:|---|"]
for k, v in R.items():
rows.append(f"| {NAMES.get(k, k)} | {v['desc']} | {v['n']:,} | **{v['acc'] * 100:.1f}%** ({v['ci95'][0] * 100:.1f}~{v['ci95'][1] * 100:.1f}) | {v['chance'] * 100:.0f}% | {v['source']} ยท {v['license']} |")
s, e = card.index("<!-- HELDOUT:start -->"), card.index("<!-- HELDOUT:end -->")
card = card[:s] + "<!-- HELDOUT:start -->\n" + "\n".join(rows) + "\n" + card[e:]
# ๊ฐ™์€ ์กฐ๊ฑด ๋น„๊ต (eval/compare/*.json, bench_compare.py ๊ฒฐ๊ณผ)
sys_path = str(ROOT / "eval")
import sys
sys.path.insert(0, sys_path)
from make_chart import comparisons
comps = comparisons()
if comps and "<!-- COMPARE:start -->" in card:
hdr = "| ๋ฒค์น˜๋งˆํฌ | **KJH-Judge-4B-ko (4B)** | " + " | ".join(n for n, _ in comps) + " |"
cmp_rows = ["**๊ฐ™์€ ์กฐ๊ฑด ๋น„๊ต** ยท ๊ฐ™์€ ๋ฌธํ•ญ ยท ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ยท 0-shot ยท ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜. ๋‹ค๋ฅธ ๋ชจ๋ธ์€ ์ƒ๊ฐ(thinking)์„ ๋„๊ณ  ๊ฐ์ž์˜ ๋Œ€ํ™” ํ˜•์‹์œผ๋กœ ๋ฌผ์—ˆ์Šต๋‹ˆ๋‹ค.", "",
hdr, "|---|---:|" + "---:|" * len(comps)]
for k, v in R.items():
cells = []
for _, c in comps:
cells.append(f"{c[k]['acc'] * 100:.1f}%" if k in c else "โ€”")
cmp_rows.append(f"| {NAMES.get(k, k)} | **{v['acc'] * 100:.1f}%** | " + " | ".join(cells) + " |")
s, e = card.index("<!-- COMPARE:start -->"), card.index("<!-- COMPARE:end -->")
card = card[:s] + "<!-- COMPARE:start -->\n" + "\n".join(cmp_rows) + "\n" + card[e:]
mi = ["model-index:", " - name: KJH-Judge-4B-ko", " results:"]
for k, v in R.items():
t, tn, ds, cfg, sp = META[k]
mi += [f" - task: {{type: {t}, name: {tn}}}",
f" dataset: {{name: {NAMES[k]}, type: {ds}" + (f", config: {cfg}" if cfg else "") + f", split: {sp}}}",
f" metrics:", f" - {{type: accuracy, value: {v['acc'] * 100:.1f}, name: accuracy (0-shot, n={v['n']})}}"]
head, body = card.split("\n---\n", 1)
head = re.sub(r"\nmodel-index:[\s\S]*$", "", head)
card = head + "\n" + "\n".join(mi) + "\n---\n" + body
if a.repo:
card = card.replace("KJH/KJH-Judge-4B-ko", a.repo)
(ROOT / "README.md").write_text(card, encoding="utf-8")
print("\n".join(rows))
if __name__ == "__main__":
main()