File size: 9,089 Bytes
4a62dcc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
"""๋‹ค๋ฅธ ๋ชจ๋ธ๊ณผ ๊ฐ™์€ ์กฐ๊ฑด์œผ๋กœ ๋น„๊ต: bench_heldout.py ์™€ ๊ฐ™์€ ๋ฐ์ดํ„ฐ ยท ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ยท ๊ฐ™์€ ์ฑ„์ (๋ณด๊ธฐ ๊ธฐํ˜ธ ์ ์ˆ˜, 0-shot).

    # OpenAI ํ˜ธํ™˜ API๋กœ ๋–  ์žˆ๋Š” ๋ชจ๋ธ (vLLM ๋“ฑ) โ€” ํ‚ค๋Š” ํ™˜๊ฒฝ ๋ณ€์ˆ˜ OPENAI_API_KEY ๋กœ
    python eval/bench_compare.py --api http://์„œ๋ฒ„:ํฌํŠธ/v1 [--api-model ๋ชจ๋ธ์ด๋ฆ„]

    # GPU ์„œ๋ฒ„์—์„œ (๋ชจ๋ธ์€ Hugging Face์—์„œ ์ž๋™์œผ๋กœ ๋ฐ›๋Š”๋‹ค)
    python eval/bench_compare.py --model Qwen/Qwen3-4B                      # ์›๋ณธ ๋ชจ๋ธ
    python eval/bench_compare.py --model google/gemma-3-27b-it --4bit       # ํฐ ๋ชจ๋ธ์€ 4bit๋กœ (bitsandbytes ํ•„์š”)
    python eval/bench_compare.py --model . --format jev                     # ์ด ๋ชจ๋ธ (ํ•™์Šต ํ˜•์‹ ๊ทธ๋Œ€๋กœ)

๊ณต์ •์„ฑ์„ ์œ„ํ•ด:
- ํ”„๋กฌํ”„ํŠธ ๋ณธ๋ฌธ(๊ณผ์ œ ์„ค๋ช… ยท ํ•ญ๋ชฉ ยท ๋ณด๊ธฐ ยท "์ •๋‹ต ๋ณด๊ธฐ์˜ ๊ธฐํ˜ธ ํ•˜๋‚˜๋งŒ ๋‹ตํ•˜์„ธ์š”.")์€ ๋ชจ๋“  ๋ชจ๋ธ์— ๋˜‘๊ฐ™๋‹ค.
- ๋Œ€ํ™” ํ‹€์€ ๊ฐ ๋ชจ๋ธ์˜ chat template ์„ ์“ฐ๊ณ , ์ƒ๊ฐ(thinking) ๊ธฐ๋Šฅ์ด ์žˆ์œผ๋ฉด ๋ˆ๋‹ค (Qwen3: enable_thinking=False).
- ๋‹ค์Œ ํ† ํฐ์—์„œ ๋ณด๊ธฐ ๊ธฐํ˜ธ ํ† ํฐ("A" ๋˜๋Š” " A" ์ค‘ ๋†’์€ ์ชฝ)์˜ ์ ์ˆ˜๋งŒ ๋น„๊ตํ•œ๋‹ค. ์ •ํ™•๋„๋Š” ์˜จ๋„ ๋ณด์ •๊ณผ ๋ฌด๊ด€ํ•˜๋‹ค.
"""
import argparse
import json
import random
import sys
import time
from pathlib import Path

import torch

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from bench_heldout import BENCH, load, wilson  # noqa: E402  ๊ฐ™์€ ๋ฐ์ดํ„ฐ ยท ๊ฐ™์€ ํ‘œ๋ณธ
from judge import chat_prefix, render_prompt  # noqa: E402  ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ ๋ณธ๋ฌธ


def build_text(tok, prompt, fmt):
    if fmt == "jev":
        return chat_prefix(prompt)
    msgs = [{"role": "user", "content": prompt}]
    try:
        return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False)
    except TypeError:
        return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)


def letter_variants(tok):
    out = []
    for i in range(26):
        L = chr(65 + i)
        ids = set()
        for s in (L, " " + L):
            e = tok.encode(s, add_special_tokens=False)
            if len(e) == 1:
                ids.add(e[0])
        out.append(sorted(ids))
    return out


@torch.no_grad()
def run(model, tok, texts, nchoices, letters, batch):
    dev = next(model.parameters()).device
    enc = [tok.encode(t, add_special_tokens=False) for t in texts]
    order = sorted(range(len(enc)), key=lambda i: len(enc[i]))
    pred = [None] * len(enc)
    pad = tok.pad_token_id if tok.pad_token_id is not None else 0
    for s in range(0, len(order), batch):
        grp = order[s:s + batch]
        n = max(len(enc[i]) for i in grp)
        x = torch.full((len(grp), n), pad, dtype=torch.long)
        m = torch.zeros((len(grp), n), dtype=torch.long)
        for r, i in enumerate(grp):  # ์˜ค๋ฅธ์ชฝ ์ฑ„์šฐ๊ธฐ: ๊ฐ ๋ฌธ์žฅ์˜ ๋งˆ์ง€๋ง‰ ์ž๋ฆฌ ์ ์ˆ˜๋งŒ ์ฝ๋Š”๋‹ค
            x[r, :len(enc[i])] = torch.tensor(enc[i]); m[r, :len(enc[i])] = 1
        logits = model(input_ids=x.to(dev), attention_mask=m.to(dev)).logits
        for r, i in enumerate(grp):
            row = logits[r, len(enc[i]) - 1].float()
            sc = [max(row[t].item() for t in letters[k]) for k in range(nchoices[i])]
            pred[i] = max(range(len(sc)), key=sc.__getitem__)
    return pred


def run_api(base, model_name, prompts, nchoices, workers=8):
    """API ๋ฐฉ์‹: ๊ฐ™์€ ํ”„๋กฌํ”„ํŠธ๋ฅผ ์‚ฌ์šฉ์ž ๋ฉ”์‹œ์ง€๋กœ ๋ณด๋‚ด๊ณ (์ƒ๊ฐ ๋”), ์ฒซ ํ† ํฐ ์ƒ์œ„ ํ™•๋ฅ ์—์„œ ํ—ˆ์šฉํ•œ ๋ณด๊ธฐ ๊ธฐํ˜ธ ์ค‘ ๊ฐ€์žฅ ๋†’์€ ๊ฒƒ์„ ๊ณ ๋ฅธ๋‹ค.
    ์ƒ์œ„ ํ™•๋ฅ ์— ๋ณด๊ธฐ ๊ธฐํ˜ธ๊ฐ€ ํ•˜๋‚˜๋„ ์—†์œผ๋ฉด ์‹ค์ œ๋กœ ๋‚ธ ๊ธ€์ž๋ฅผ ์“ด๋‹ค."""
    import concurrent.futures as cf
    import os
    import urllib.request
    key = os.environ.get("OPENAI_API_KEY", "")
    hdr = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})}

    def one(i):
        body = {"model": model_name, "messages": [{"role": "user", "content": prompts[i]}], "max_tokens": 1, "temperature": 0,
                "logprobs": True, "top_logprobs": 20, "chat_template_kwargs": {"enable_thinking": False}}
        for k in range(8):
            try:
                req = urllib.request.Request(base + "/chat/completions", json.dumps(body, ensure_ascii=False).encode(), hdr)
                c = json.loads(urllib.request.urlopen(req, timeout=120).read())["choices"][0]
                allowed = {chr(65 + j): j for j in range(nchoices[i])}
                best = None
                for t in (c.get("logprobs") or {}).get("content", [{}])[0].get("top_logprobs", []):
                    L = t["token"].strip()
                    if L in allowed and (best is None or t["logprob"] > best[1]):
                        best = (allowed[L], t["logprob"])
                if best:
                    return best[0]
                txt = (c["message"].get("content") or "").strip()[:1]
                return allowed.get(txt, -1)
            except urllib.error.HTTPError as e:
                if e.code in (400, 401, 403, 404):
                    raise
                time.sleep(2)
            except Exception:
                time.sleep(2)
        return -1
    with cf.ThreadPoolExecutor(workers) as ex:
        return list(ex.map(one, range(len(prompts))))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", help="Hugging Face ๋ชจ๋ธ id ๋˜๋Š” ํด๋”")
    ap.add_argument("--api", help="OpenAI ํ˜ธํ™˜ API ์ฃผ์†Œ (โ€ฆ/v1). ํ‚ค๋Š” ํ™˜๊ฒฝ ๋ณ€์ˆ˜ OPENAI_API_KEY")
    ap.add_argument("--api-model", help="API์˜ ๋ชจ๋ธ ์ด๋ฆ„ (์ƒ๋žตํ•˜๋ฉด /models ์˜ ์ฒซ ๋ฒˆ์งธ)")
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--format", choices=["chat", "jev"], default="chat", help="jev = ์ด ๋ชจ๋ธ์˜ ํ•™์Šต ํ˜•์‹ (KJH-Judge ์ „์šฉ)")
    ap.add_argument("--4bit", dest="q4", action="store_true", help="4bit ์–‘์žํ™”๋กœ ๋ถˆ๋Ÿฌ์˜ค๊ธฐ (ํฐ ๋ชจ๋ธ์šฉ, bitsandbytes ํ•„์š”)")
    ap.add_argument("--n", type=int, default=1000)
    ap.add_argument("--only", nargs="*")
    ap.add_argument("--batch", type=int, default=8)
    a = ap.parse_args()
    if not (a.model or a.api):
        ap.error("--model ๋˜๋Š” --api ๊ฐ€ ํ•„์š”ํ•ฉ๋‹ˆ๋‹ค")
    if a.api:
        import os
        import urllib.request
        a.api = a.api.rstrip("/")
        if not a.api_model:
            key = os.environ.get("OPENAI_API_KEY", "")
            req = urllib.request.Request(a.api + "/models", headers={"Authorization": f"Bearer {key}"} if key else {})
            a.api_model = json.loads(urllib.request.urlopen(req, timeout=30).read())["data"][0]["id"]
        name = "api-" + a.api_model.replace("/", "_")
        tok = model = letters = None
    else:
        from transformers import AutoModelForCausalLM, AutoTokenizer
        tok = AutoTokenizer.from_pretrained(a.model)
        try:
            import accelerate  # noqa: F401
            kw = {"device_map": "auto", "dtype": torch.bfloat16}
        except ImportError:  # accelerate ๊ฐ€ ์—†์œผ๋ฉด ํ•œ ์žฅ์น˜์— ์ง์ ‘ ์˜ฌ๋ฆฐ๋‹ค
            kw = {"dtype": torch.bfloat16 if torch.cuda.is_available() else torch.float32}
        if a.q4:
            from transformers import BitsAndBytesConfig
            kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4")
        model = AutoModelForCausalLM.from_pretrained(a.model, **kw).eval()
        if "device_map" not in kw:
            model = model.to("cuda" if torch.cuda.is_available() else "cpu")
        letters = letter_variants(tok)
        name = Path(a.model).resolve().name if Path(a.model).exists() else a.model.split("/")[-1]
    out = HERE / "compare" / f"{name}{'-4bit' if a.q4 else ''}{'-' + a.format if a.format == 'jev' else ''}.json"
    out.parent.mkdir(exist_ok=True)
    res = {}
    for b in (a.only or BENCH):
        t0 = time.time()
        data = load(b, random.Random(f"kjh-judge:{b}"), a.n)  # bench_heldout.py ์™€ ๊ฐ™์€ ์‹œ๋“œ โ†’ ๊ฐ™์€ ๋ฌธํ•ญ
        prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r, _ in data]
        nch = [len(r["choices"]) for r, _ in data]
        if a.api:
            pred = run_api(a.api, a.api_model, prompts, nch, a.workers)
        else:
            pred = run(model, tok, [build_text(tok, p, a.format) for p in prompts], nch, letters, a.batch)
        k = sum(p == g for p, (_, g) in zip(pred, data))
        lo, hi = wilson(k, len(data))
        res[b] = {"n": len(data), "acc": round(k / len(data), 4), "ci95": [round(lo, 4), round(hi, 4)], "seconds": round(time.time() - t0)}
        print(f"{b:<10} n={len(data):>5}  ์ •ํ™•๋„ {k / len(data):6.1%}  ({lo:.1%}~{hi:.1%})  {res[b]['seconds']}s", flush=True)
        out.write_text(json.dumps({"model": a.api_model if a.api else a.model, "via": "api" if a.api else "local", "format": a.format, "4bit": a.q4, "made": time.strftime("%Y-%m-%d %H:%M"), "results": res}, ensure_ascii=False, indent=1), encoding="utf-8")
    print("์ €์žฅ:", out)


if __name__ == "__main__":
    main()