File size: 4,076 Bytes
4a62dcc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
"""eval/heldout_results.json ์œผ๋กœ README.md ๋ฅผ ์ฑ„์šด๋‹ค.
 - ๋ณธ๋ฌธ์˜ 'ํ•™์Šตํ•˜์ง€ ์•Š์€ ๊ณต๊ฐœ ๋ฒค์น˜๋งˆํฌ' ํ‘œ
 - ๋งจ ์œ„ YAML ์˜ model-index (๋ชจ๋ธ ํŽ˜์ด์ง€ ์˜ค๋ฅธ์ชฝ 'Evaluation results' ์ƒ์ž)

    python eval/fill_card.py [--repo ๊ณ„์ •/KJH-Judge-4B-ko]
"""
import argparse
import json
import re
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
NAMES = {"nsmc": "NSMC", "kornli": "KorNLI (XNLI)", "klue_sts": "KLUE-STS", "unsmile": "Korean UnSmile", "kmmlu": "KMMLU", "haerae": "HAE-RAE Bench 1.0"}
META = {  # model-index: (๊ณผ์ œ ์ข…๋ฅ˜, ๊ณผ์ œ ์ด๋ฆ„, ๋ฐ์ดํ„ฐ์…‹ id, ์„ค์ •, ๋ถ„ํ• )
    "nsmc": ("text-classification", "Sentiment classification", "e9t/nsmc", None, "test"),
    "kornli": ("text-classification", "Natural language inference", "kakaobrain/kor_nli", "xnli", "test"),
    "klue_sts": ("text-classification", "Paraphrase detection (binary STS)", "klue/klue", "sts", "validation"),
    "unsmile": ("text-classification", "Hate speech detection", "smilegate-ai/kor_unsmile", None, "valid"),
    "kmmlu": ("multiple-choice", "Knowledge QA", "HAERAE-HUB/KMMLU", None, "test"),
    "haerae": ("multiple-choice", "Korean knowledge QA", "HAERAE-HUB/HAE_RAE_BENCH_1.0", None, "test"),
}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--repo", help="๊ณ„์ •/์ €์žฅ์†Œ ์ด๋ฆ„. ์ฃผ๋ฉด ์นด๋“œ ์•ˆ์˜ KJH/KJH-Judge-4B-ko ๋ฅผ ๋ฐ”๊พผ๋‹ค")
    a = ap.parse_args()
    R = json.loads((ROOT / "eval" / "heldout_results.json").read_text(encoding="utf-8"))["results"]
    card = (ROOT / "README.md").read_text(encoding="utf-8")

    rows = ["| ๋ฒค์น˜๋งˆํฌ | ๋ฌด์—‡์„ ์žฌ๋‚˜ | ๋ฌธํ•ญ | ์ •ํ™•๋„ (95% ๊ตฌ๊ฐ„) | ๋ฌด์ž‘์œ„ | ์ถœ์ฒ˜ ยท ๋ผ์ด์„ ์Šค |", "|---|---|---:|---:|---:|---|"]
    for k, v in R.items():
        rows.append(f"| {NAMES.get(k, k)} | {v['desc']} | {v['n']:,} | **{v['acc'] * 100:.1f}%** ({v['ci95'][0] * 100:.1f}~{v['ci95'][1] * 100:.1f}) | {v['chance'] * 100:.0f}% | {v['source']} ยท {v['license']} |")
    s, e = card.index("<!-- HELDOUT:start -->"), card.index("<!-- HELDOUT:end -->")
    card = card[:s] + "<!-- HELDOUT:start -->\n" + "\n".join(rows) + "\n" + card[e:]

    # ๊ฐ™์€ ์กฐ๊ฑด ๋น„๊ต (eval/compare/*.json, bench_compare.py ๊ฒฐ๊ณผ)
    sys_path = str(ROOT / "eval")
    import sys
    sys.path.insert(0, sys_path)
    from make_chart import comparisons
    comps = comparisons()
    if comps and "<!-- COMPARE:start -->" in card:
        hdr = "| ๋ฒค์น˜๋งˆํฌ | **KJH-Judge-4B-ko (4B)** | " + " | ".join(n for n, _ in comps) + " |"
        cmp_rows = ["**๊ฐ™์€ ์กฐ๊ฑด ๋น„๊ต** ยท ๊ฐ™์€ ๋ฌธํ•ญ ยท ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ยท 0-shot ยท ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜. ๋‹ค๋ฅธ ๋ชจ๋ธ์€ ์ƒ๊ฐ(thinking)์„ ๋„๊ณ  ๊ฐ์ž์˜ ๋Œ€ํ™” ํ˜•์‹์œผ๋กœ ๋ฌผ์—ˆ์Šต๋‹ˆ๋‹ค.", "",
                    hdr, "|---|---:|" + "---:|" * len(comps)]
        for k, v in R.items():
            cells = []
            for _, c in comps:
                cells.append(f"{c[k]['acc'] * 100:.1f}%" if k in c else "โ€”")
            cmp_rows.append(f"| {NAMES.get(k, k)} | **{v['acc'] * 100:.1f}%** | " + " | ".join(cells) + " |")
        s, e = card.index("<!-- COMPARE:start -->"), card.index("<!-- COMPARE:end -->")
        card = card[:s] + "<!-- COMPARE:start -->\n" + "\n".join(cmp_rows) + "\n" + card[e:]

    mi = ["model-index:", "  - name: KJH-Judge-4B-ko", "    results:"]
    for k, v in R.items():
        t, tn, ds, cfg, sp = META[k]
        mi += [f"      - task: {{type: {t}, name: {tn}}}",
               f"        dataset: {{name: {NAMES[k]}, type: {ds}" + (f", config: {cfg}" if cfg else "") + f", split: {sp}}}",
               f"        metrics:", f"          - {{type: accuracy, value: {v['acc'] * 100:.1f}, name: accuracy (0-shot, n={v['n']})}}"]
    head, body = card.split("\n---\n", 1)
    head = re.sub(r"\nmodel-index:[\s\S]*$", "", head)
    card = head + "\n" + "\n".join(mi) + "\n---\n" + body
    if a.repo:
        card = card.replace("KJH/KJH-Judge-4B-ko", a.repo)
    (ROOT / "README.md").write_text(card, encoding="utf-8")
    print("\n".join(rows))


if __name__ == "__main__":
    main()