File size: 5,187 Bytes
4a62dcc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
"""์†๋„ ๋น„๊ต: ๊ฐ™์€ ๋ฌธํ•ญ์„ ๊ฐ™์€ PC์—์„œ ๋ณด๋‚ด ๊ฑธ๋ฆฐ ์‹œ๊ฐ„์„ ์žฐ๋‹ค (๋„คํŠธ์›Œํฌ ์™•๋ณต ํฌํ•จ).

    $env:OPENAI_API_KEY = "โ€ฆ"     # ๋น„๊ต ๋ชจ๋ธ API ํ‚ค
    python eval/bench_speed.py --kjh http://์„œ๋ฒ„:32500 --api http://์„œ๋ฒ„:30102/v1

์ธก์ •
 1. ํŒ์ • 1ํšŒ ์ง€์—ฐ: ๋ฌธํ•ญ์„ ํ•˜๋‚˜์”ฉ ๋ณด๋‚ด ์‘๋‹ต๊นŒ์ง€ ๊ฑธ๋ฆฐ ์‹œ๊ฐ„ (KJH: /api/decide, ๋น„๊ต ๋ชจ๋ธ: ์ƒ๊ฐ ๋„๊ณ  max_tokens=1)
 2. ๋™์‹œ ์ฒ˜๋ฆฌ๋Ÿ‰: ๋™์‹œ์— N๊ฐœ์”ฉ ๋ณด๋‚ด ์ดˆ๋‹น ํŒ์ • ์ˆ˜
 3. ๋น„๊ต ๋ชจ๋ธ์„ ํ‰์†Œ์ฒ˜๋Ÿผ ์“ธ ๋•Œ: ์ƒ๊ฐ์„ ์ผœ๊ณ  ๋‹ต์„ ์ƒ์„ฑํ•˜๊ฒŒ ํ–ˆ์„ ๋•Œ ์‹œ๊ฐ„๊ณผ ์ƒ์„ฑ ํ† ํฐ ์ˆ˜ (์ผ๋ถ€ ๋ฌธํ•ญ)
"""
import argparse
import concurrent.futures as cf
import json
import os
import random
import statistics
import sys
import time
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from bench_heldout import load  # noqa: E402
from judge import render_prompt  # noqa: E402


RETRIES = {"n": 0}


def post(url, body, hdr, timeout=300):
    """์—ฐ๊ฒฐ์ด ์ž ๊น ๊ฑฐ๋ถ€๋˜๋ฉด ๋‹ค์‹œ ์‹œ๋„ํ•œ๋‹ค. ์žฌ์‹œ๋„์— ๊ฑธ๋ฆฐ ์‹œ๊ฐ„์€ ๊ทธ ์š”์ฒญ ์‹œ๊ฐ„์— ํฌํ•จ๋˜๋ฏ€๋กœ ์žฌ์‹œ๋„ ํšŸ์ˆ˜๋ฅผ ํ•จ๊ป˜ ๊ธฐ๋กํ•œ๋‹ค."""
    req = urllib.request.Request(url, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json", **hdr})
    for k in range(10):
        try:
            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())
        except urllib.error.HTTPError:
            raise
        except Exception:
            if k == 9:
                raise
            RETRIES["n"] += 1
            time.sleep(1.5)


def stat(ms):
    s = sorted(ms)
    return {"n": len(s), "median_ms": round(statistics.median(s)), "p90_ms": round(s[int(len(s) * 0.9) - 1]), "mean_ms": round(statistics.mean(s))}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--kjh", required=True, help="KJH ํŒ์ • ์„œ๋ฒ„ (studio/serve.py)")
    ap.add_argument("--api", required=True, help="๋น„๊ต ๋ชจ๋ธ OpenAI ํ˜ธํ™˜ ์ฃผ์†Œ (โ€ฆ/v1)")
    ap.add_argument("--api-model")
    ap.add_argument("--n", type=int, default=60)
    ap.add_argument("--conc", type=int, default=4)
    ap.add_argument("--think-n", type=int, default=10)
    ap.add_argument("--out", default=str(HERE / "speed_results.json"))
    a = ap.parse_args()
    kjh, api = a.kjh.rstrip("/"), a.api.rstrip("/")
    key = os.environ.get("OPENAI_API_KEY", "")
    hdr = {"Authorization": f"Bearer {key}"} if key else {}
    if not a.api_model:
        a.api_model = json.loads(urllib.request.urlopen(urllib.request.Request(api + "/models", headers=hdr), timeout=30).read())["data"][0]["id"]
    rng = random.Random("speed")
    data = load("nsmc", rng, a.n // 2) + load("kornli", rng, a.n - a.n // 2)
    reqs = [r for r, _ in data]
    prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r in reqs]

    def kjh_one(i):
        t = time.perf_counter()
        r = post(kjh + "/api/decide", {"instruction": reqs[i]["instruction"], "item": reqs[i]["item"], "choices": [{"text": c} for c in reqs[i]["choices"]]}, {})
        return (time.perf_counter() - t) * 1000, r.get("ms")

    def api_one(i, think=False):
        body = {"model": a.api_model, "messages": [{"role": "user", "content": prompts[i]}], "temperature": 0,
                "chat_template_kwargs": {"enable_thinking": think}}
        body.update({"max_tokens": 4096} if think else {"max_tokens": 1, "logprobs": True, "top_logprobs": 20})
        t = time.perf_counter()
        r = post(api + "/chat/completions", body, hdr, timeout=600)
        return (time.perf_counter() - t) * 1000, (r.get("usage") or {}).get("completion_tokens")

    for f in (kjh_one, api_one):  # ์˜ˆ์—ด
        f(0)
    out = {"made": time.strftime("%Y-%m-%d %H:%M"), "api_model": a.api_model, "n": len(reqs), "note": "๊ฐ™์€ PC์—์„œ ์ธก์ •, ๋„คํŠธ์›Œํฌ ์™•๋ณต ํฌํ•จ"}
    k = [kjh_one(i) for i in range(len(reqs))]
    out["kjh_single"] = {**stat([x[0] for x in k]), "server_compute_median_ms": round(statistics.median([x[1] for x in k if x[1] is not None]))}
    print("KJH 1ํšŒ", out["kjh_single"], flush=True)
    b = [api_one(i)[0] for i in range(len(reqs))]
    out["api_single"] = stat(b)
    print("๋น„๊ต 1ํšŒ", out["api_single"], flush=True)
    for name, fn in (("kjh_throughput", kjh_one), ("api_throughput", api_one)):
        t = time.perf_counter()
        with cf.ThreadPoolExecutor(a.conc) as ex:
            list(ex.map(fn, range(len(reqs))))
        el = time.perf_counter() - t
        out[name] = {"concurrency": a.conc, "decisions": len(reqs), "seconds": round(el, 2), "per_sec": round(len(reqs) / el, 1)}
        print(name, out[name], flush=True)
    th = [api_one(i, True) for i in range(min(a.think_n, len(reqs)))]
    out["api_thinking"] = {**stat([x[0] for x in th]), "completion_tokens_median": statistics.median([x[1] or 0 for x in th])}
    print("๋น„๊ต ์ƒ๊ฐ ์ผฌ", out["api_thinking"], flush=True)
    out["connection_retries"] = RETRIES["n"]
    Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
    print("์ €์žฅ:", a.out)


if __name__ == "__main__":
    main()