File size: 7,492 Bytes
4a62dcc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
"""๋ชจ๋ธ ์นด๋“œ์šฉ ๋ฒค์น˜๋งˆํฌ ๊ทธ๋ฆผ(assets/benchmark.png)์„ ๋งŒ๋“ ๋‹ค: HTML๋กœ ๊ทธ๋ฆฐ ๋’ค Edge/Chrome ํ—ค๋“œ๋ฆฌ์Šค๋กœ ์ฐ๋Š”๋‹ค.

    python eval/make_chart.py [--browser ๊ฒฝ๋กœ]
"""
import argparse
import json
import shutil
import subprocess
import tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
NAMES = {"nsmc": ("NSMC", "์˜ํ™” ๋ฆฌ๋ทฐ ๊ฐ์„ฑ"), "klue_sts": ("KLUE-STS", "๋‘ ๋ฌธ์žฅ์ด ๊ฐ™์€ ๋œป์ธ๊ฐ€"), "unsmile": ("Korean UnSmile", "์•…ํ”Œยทํ˜์˜ค ํ‘œํ˜„ ํŒ๋ณ„"),
         "kornli": ("KorNLI", "๋ฌธ์žฅ ๊ด€๊ณ„ 3๋ถ„๋ฅ˜"), "haerae": ("HAE-RAE 1.0", "ํ•œ๊ตญ ๋ฌธํ™”ยท์–ดํœ˜ ์ง€์‹"), "kmmlu": ("KMMLU", "์ „๋ฌธ ์ง€์‹ 4์ง€์„ ๋‹ค")}
KNOWLEDGE = {"kmmlu", "haerae"}


def speed_box():
    f = ROOT / "eval" / "speed_results.json"
    if not f.is_file():
        return '<div class="box"><b>~75ms</b><span>ํŒ์ • ํ•œ ๋ฒˆ (GPU, ์งง์€ ์ž…๋ ฅ)<br>๊ธ€์„ ์ƒ์„ฑํ•˜์ง€ ์•Š๊ณ  ํ•œ ๋ฒˆ๋งŒ ๊ณ„์‚ฐ</span></div>'
    s = json.loads(f.read_text(encoding="utf-8"))
    return (f'<div class="box"><b>{s["kjh_single"]["median_ms"]}ms</b><span>ํŒ์ • ํ•œ ๋ฒˆ (์ค‘์•™๊ฐ’, ์™•๋ณต ํฌํ•จ)<br>'
            f'Gemma 4 26B-A4B {s["api_single"]["median_ms"]}ms ยท ์ƒ๊ฐ ์ผœ๊ณ  ๋‹ตํ•˜๋ฉด {s["api_thinking"]["median_ms"] / 1000:.1f}์ดˆ</span></div>')


def comparisons():
    """eval/compare/*.json (bench_compare.py ๊ฒฐ๊ณผ) โ†’ [(์ด๋ฆ„, ๊ฒฐ๊ณผ)]. KJH ์ž์‹ (-jev)์€ ๋บ€๋‹ค."""
    d = ROOT / "eval" / "compare"
    labels = json.loads((d / "labels.json").read_text(encoding="utf-8")) if (d / "labels.json").is_file() else {}
    out = []
    for f in sorted(d.glob("*.json")):
        if f.name == "labels.json" or f.stem.endswith("-jev"):
            continue
        j = json.loads(f.read_text(encoding="utf-8"))
        out.append((labels.get(f.stem, j.get("model", f.stem)), j["results"]))
    return out


def html(R):
    order = [k for k in NAMES if k in R]
    comps = comparisons()
    rows = ""
    for k in order:
        v, (nm, desc) = R[k], NAMES[k]
        acc, lo, hi, ch = v["acc"] * 100, v["ci95"][0] * 100, v["ci95"][1] * 100, v["chance"] * 100
        cls = "know" if k in KNOWLEDGE else ""
        cmp_html = "".join(f'<div class="cmp" style="width:{c[k]["acc"] * 100}%"><i>{c[k]["acc"] * 100:.1f}</i></div>' for _, c in comps if k in c)
        rows += f"""<div class="row {cls} {'two' if cmp_html else ''}"><div class="nm"><b>{nm}</b><span>{desc}</span></div>
          <div class="trk"><div class="track"><div class="bar" style="width:{acc}%"></div><div class="ci" style="left:{lo}%;width:{hi - lo}%"></div>
          <div class="chance" style="left:{ch}%"><i>๋ฌด์ž‘์œ„ {ch:.0f}%</i></div></div>{cmp_html}</div><div class="val">{acc:.1f}<small>%</small></div></div>"""
    legend_cmp = "".join(f'<span><i style="background:#cbd5e1;border:1px solid #94a3b8"></i>{n}</span>' for n, _ in comps)
    return f"""<!doctype html><html><head><meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans+KR:wght@400;500;600;700&family=IBM+Plex+Mono:wght@500;600&display=swap">
<style>
*{{box-sizing:border-box}}body{{margin:0;background:#ffffff;font-family:'IBM Plex Sans KR','Malgun Gothic',sans-serif;color:#121823}}
.wrap{{width:1200px;padding:40px 48px 36px}}
h1{{margin:0;font-size:30px;font-weight:700;letter-spacing:-.02em}}h1 span{{color:#1d4ed8}}
.sub{{margin:8px 0 28px;color:#5a6476;font-size:16px}}
.grid{{display:grid;grid-template-columns:1fr 270px;gap:36px}}
.row{{display:grid;grid-template-columns:210px 1fr 96px;gap:16px;align-items:center;height:62px}}
.nm b{{display:block;font-size:17px}}.nm span{{font-size:13px;color:#5a6476}}
.track{{position:relative;height:26px;background:#eef1f5;border-radius:5px}}
.bar{{position:absolute;inset:0 auto 0 0;background:#1d4ed8;border-radius:5px}}
.know .bar{{background:#94a3b8}}
.ci{{position:absolute;top:11px;height:4px;background:rgba(255,255,255,.75);border-radius:2px}}
.chance{{position:absolute;top:-6px;bottom:-6px;border-left:2px dashed #121823;opacity:.55}}
.chance i{{position:absolute;top:-17px;left:4px;font-style:normal;font-size:11px;white-space:nowrap;color:#121823}}
.val{{font:600 26px 'IBM Plex Mono',monospace;text-align:right}}.val small{{font-size:15px;color:#5a6476}}
.sep{{margin:6px 0 4px;font-size:12.5px;color:#5a6476;padding-left:226px}}
.side{{display:flex;flex-direction:column;gap:12px}}
.box{{border:1px solid #d8dde5;border-radius:10px;padding:14px 16px}}
.box b{{display:block;font:600 28px 'IBM Plex Mono',monospace;letter-spacing:-.02em}}.box span{{font-size:13px;color:#5a6476;line-height:1.45;display:block}}
.foot{{margin-top:22px;font-size:12.5px;color:#5a6476}}
.legend{{display:flex;flex-wrap:wrap;gap:6px 18px;font-size:13px;color:#5a6476;margin-bottom:10px;padding-left:226px}}.legend i{{display:inline-block;width:12px;height:12px;border-radius:3px;margin-right:6px;vertical-align:-1px}}
.row.two{{height:78px}}.trk{{display:flex;flex-direction:column;gap:5px}}
.cmp{{position:relative;height:10px;background:#cbd5e1;border-radius:3px;max-width:100%}}
.cmp i{{position:absolute;left:calc(100% + 6px);top:-4px;font:500 11.5px 'IBM Plex Mono',monospace;color:#5a6476;font-style:normal}}
</style></head><body><div class="wrap">
<h1><span>KJH-Judge-4B-ko</span> ยท ํ•™์Šตํ•˜์ง€ ์•Š์€ ํ•œ๊ตญ์–ด ๋ฒค์น˜๋งˆํฌ</h1>
<div class="sub">ํ•™์Šต ๋ฐ์ดํ„ฐ์™€ ๋‹ค๋ฅธ ์ถœ์ฒ˜์˜ ๊ณต๊ฐœ ๋ฐ์ดํ„ฐ, 0-shot, ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜ ๋ฐฉ์‹. ๋ง‰๋Œ€ ์•ˆ ํฐ ์„ ์€ 95% ์‹ ๋ขฐ๊ตฌ๊ฐ„.</div>
<div class="grid"><div>
<div class="legend"><span><i style="background:#1d4ed8"></i>KJH-Judge-4B-ko ยท ํŒ์ • ๊ณผ์ œ</span><span><i style="background:#94a3b8"></i>KJH-Judge-4B-ko ยท ์ง€์‹ ์‹œํ—˜</span>{legend_cmp}</div>
{rows}</div>
<div class="side">
<div class="box"><b>4.02B</b><span>ํŒŒ๋ผ๋ฏธํ„ฐ ยท Qwen3-4B์—์„œ ๋ฏธ์„ธ์กฐ์ •</span></div>
{speed_box()}
<div class="box"><b>0.027</b><span>๊ธฐ๋Œ€ ๋ณด์ • ์˜ค์ฐจ(ECE)<br>ํ™•๋ฅ ์„ ๊ทธ๋Œ€๋กœ ๊ธฐ์ค€์œผ๋กœ ์“ธ ์ˆ˜ ์žˆ์Œ</span></div>
<div class="box"><b>97.9%</b><span>๋ณด๊ธฐ๋ฅผ ๊ฑฐ๊พธ๋กœ ๋†“์•„๋„ ๊ฐ™์€ ๋‹ต</span></div>
</div></div>
<div class="foot">๋ฌธํ•ญ ์ˆ˜: NSMCยทKorNLIยทUnSmile ๊ฐ 1,000 (๊ณ ์ • ์‹œ๋“œ ํ‘œ๋ณธ), KLUE-STS 519 (์ „์ฒด), KMMLU {R.get('kmmlu', {}).get('n', 0):,} (45๊ณผ๋ชฉ ๊ณ ๋ฅด๊ฒŒ), HAE-RAE {R.get('haerae', {}).get('n', 0):,} (6์˜์—ญ ๊ณ ๋ฅด๊ฒŒ). ์žฌํ˜„: eval/bench_heldout.py</div>
</div></body></html>"""


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--browser", default=next((p for p in [r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe", r"C:\Program Files\Google\Chrome\Application\chrome.exe",
                                                             shutil.which("google-chrome") or "", shutil.which("chromium") or ""] if p and Path(p).exists()), None))
    a = ap.parse_args()
    R = json.loads((ROOT / "eval" / "heldout_results.json").read_text(encoding="utf-8"))["results"]
    (ROOT / "assets").mkdir(exist_ok=True)
    src = ROOT / "eval" / "chart.html"
    src.write_text(html(R), encoding="utf-8")
    out = ROOT / "assets" / "benchmark.png"
    h = 250 + (78 if comparisons() else 62) * len(R) + 40
    with tempfile.TemporaryDirectory() as prof:
        subprocess.run([a.browser, "--headless=new", "--disable-gpu", f"--user-data-dir={prof}", "--hide-scrollbars", "--force-device-scale-factor=2",
                        f"--window-size=1200,{h}", "--virtual-time-budget=4000", f"--screenshot={out}", src.as_uri()], check=True, timeout=120)
    print("์ €์žฅ:", out)


if __name__ == "__main__":
    main()