Multiple Choice
Transformers
Safetensors
Korean
qwen3
text-generation
korean
classification
natural-language-inference
function-calling
tool-use
calibrated
Eval Results (legacy)
text-generation-inference
Instructions to use coderecode95/KJH-Judge-4B-ko with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use coderecode95/KJH-Judge-4B-ko with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("coderecode95/KJH-Judge-4B-ko") model = AutoModelForCausalLM.from_pretrained("coderecode95/KJH-Judge-4B-ko", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 5,187 Bytes
4a62dcc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """์๋ ๋น๊ต: ๊ฐ์ ๋ฌธํญ์ ๊ฐ์ PC์์ ๋ณด๋ด ๊ฑธ๋ฆฐ ์๊ฐ์ ์ฐ๋ค (๋คํธ์ํฌ ์๋ณต ํฌํจ).
$env:OPENAI_API_KEY = "โฆ" # ๋น๊ต ๋ชจ๋ธ API ํค
python eval/bench_speed.py --kjh http://์๋ฒ:32500 --api http://์๋ฒ:30102/v1
์ธก์
1. ํ์ 1ํ ์ง์ฐ: ๋ฌธํญ์ ํ๋์ฉ ๋ณด๋ด ์๋ต๊น์ง ๊ฑธ๋ฆฐ ์๊ฐ (KJH: /api/decide, ๋น๊ต ๋ชจ๋ธ: ์๊ฐ ๋๊ณ max_tokens=1)
2. ๋์ ์ฒ๋ฆฌ๋: ๋์์ N๊ฐ์ฉ ๋ณด๋ด ์ด๋น ํ์ ์
3. ๋น๊ต ๋ชจ๋ธ์ ํ์์ฒ๋ผ ์ธ ๋: ์๊ฐ์ ์ผ๊ณ ๋ต์ ์์ฑํ๊ฒ ํ์ ๋ ์๊ฐ๊ณผ ์์ฑ ํ ํฐ ์ (์ผ๋ถ ๋ฌธํญ)
"""
import argparse
import concurrent.futures as cf
import json
import os
import random
import statistics
import sys
import time
import urllib.request
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from bench_heldout import load # noqa: E402
from judge import render_prompt # noqa: E402
RETRIES = {"n": 0}
def post(url, body, hdr, timeout=300):
"""์ฐ๊ฒฐ์ด ์ ๊น ๊ฑฐ๋ถ๋๋ฉด ๋ค์ ์๋ํ๋ค. ์ฌ์๋์ ๊ฑธ๋ฆฐ ์๊ฐ์ ๊ทธ ์์ฒญ ์๊ฐ์ ํฌํจ๋๋ฏ๋ก ์ฌ์๋ ํ์๋ฅผ ํจ๊ป ๊ธฐ๋กํ๋ค."""
req = urllib.request.Request(url, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json", **hdr})
for k in range(10):
try:
return json.loads(urllib.request.urlopen(req, timeout=timeout).read())
except urllib.error.HTTPError:
raise
except Exception:
if k == 9:
raise
RETRIES["n"] += 1
time.sleep(1.5)
def stat(ms):
s = sorted(ms)
return {"n": len(s), "median_ms": round(statistics.median(s)), "p90_ms": round(s[int(len(s) * 0.9) - 1]), "mean_ms": round(statistics.mean(s))}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--kjh", required=True, help="KJH ํ์ ์๋ฒ (studio/serve.py)")
ap.add_argument("--api", required=True, help="๋น๊ต ๋ชจ๋ธ OpenAI ํธํ ์ฃผ์ (โฆ/v1)")
ap.add_argument("--api-model")
ap.add_argument("--n", type=int, default=60)
ap.add_argument("--conc", type=int, default=4)
ap.add_argument("--think-n", type=int, default=10)
ap.add_argument("--out", default=str(HERE / "speed_results.json"))
a = ap.parse_args()
kjh, api = a.kjh.rstrip("/"), a.api.rstrip("/")
key = os.environ.get("OPENAI_API_KEY", "")
hdr = {"Authorization": f"Bearer {key}"} if key else {}
if not a.api_model:
a.api_model = json.loads(urllib.request.urlopen(urllib.request.Request(api + "/models", headers=hdr), timeout=30).read())["data"][0]["id"]
rng = random.Random("speed")
data = load("nsmc", rng, a.n // 2) + load("kornli", rng, a.n - a.n // 2)
reqs = [r for r, _ in data]
prompts = [render_prompt(r["instruction"], r["item"], r["choices"]) for r in reqs]
def kjh_one(i):
t = time.perf_counter()
r = post(kjh + "/api/decide", {"instruction": reqs[i]["instruction"], "item": reqs[i]["item"], "choices": [{"text": c} for c in reqs[i]["choices"]]}, {})
return (time.perf_counter() - t) * 1000, r.get("ms")
def api_one(i, think=False):
body = {"model": a.api_model, "messages": [{"role": "user", "content": prompts[i]}], "temperature": 0,
"chat_template_kwargs": {"enable_thinking": think}}
body.update({"max_tokens": 4096} if think else {"max_tokens": 1, "logprobs": True, "top_logprobs": 20})
t = time.perf_counter()
r = post(api + "/chat/completions", body, hdr, timeout=600)
return (time.perf_counter() - t) * 1000, (r.get("usage") or {}).get("completion_tokens")
for f in (kjh_one, api_one): # ์์ด
f(0)
out = {"made": time.strftime("%Y-%m-%d %H:%M"), "api_model": a.api_model, "n": len(reqs), "note": "๊ฐ์ PC์์ ์ธก์ , ๋คํธ์ํฌ ์๋ณต ํฌํจ"}
k = [kjh_one(i) for i in range(len(reqs))]
out["kjh_single"] = {**stat([x[0] for x in k]), "server_compute_median_ms": round(statistics.median([x[1] for x in k if x[1] is not None]))}
print("KJH 1ํ", out["kjh_single"], flush=True)
b = [api_one(i)[0] for i in range(len(reqs))]
out["api_single"] = stat(b)
print("๋น๊ต 1ํ", out["api_single"], flush=True)
for name, fn in (("kjh_throughput", kjh_one), ("api_throughput", api_one)):
t = time.perf_counter()
with cf.ThreadPoolExecutor(a.conc) as ex:
list(ex.map(fn, range(len(reqs))))
el = time.perf_counter() - t
out[name] = {"concurrency": a.conc, "decisions": len(reqs), "seconds": round(el, 2), "per_sec": round(len(reqs) / el, 1)}
print(name, out[name], flush=True)
th = [api_one(i, True) for i in range(min(a.think_n, len(reqs)))]
out["api_thinking"] = {**stat([x[0] for x in th]), "completion_tokens_median": statistics.median([x[1] or 0 for x in th])}
print("๋น๊ต ์๊ฐ ์ผฌ", out["api_thinking"], flush=True)
out["connection_retries"] = RETRIES["n"]
Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
print("์ ์ฅ:", a.out)
if __name__ == "__main__":
main()
|