Kev-0.8B-RKNN3-RK1828 / host /e2e_check.py
jj90's picture
Add model card, deployment guide and host-side code
a846b3d verified
Raw History Blame Contribute Delete
7.08 kB
"""端到端验证在线服务:从 decision-v2 取题 → 走 HTTP → 与标签比。
这一步验的是"服务路径"而不是"批量路径":
1. kev_serve 里的 render/option_text 与 plan_a 是否一致(不一致准确率会掉)
2. score 题型能不能跑通(此前只测过 choice/noul)
3. 三种题型混在一起时的逐题正确率
用法:在 Windows 上运行(服务与 daemon 都在本机)。
python e2e_check.py [每题最多选项数] [最多记录数]
"""
import json
import os
import random
import subprocess
import sys
import time
import urllib.request
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
HERE = os.path.dirname(os.path.abspath(__file__))
PORT = random.randint(30000, 39999)
# 数据集默认 Windows 本机;在其他机器上跑时用 KEV_DEV 指过去
DEV = os.environ.get("KEV_DEV") or r"D:\kevsrc\evals\decision-v2\development.jsonl"
# 给了 KEV_URL 就直接连**已有服务**,不再自己起进程 ——
# 这样可以从另一台机器(走 SSH 端口转发)验证同一个实例,
# 而不是每次测都新起一个(新起的不代表线上那个的配置)。
EXTERNAL_URL = os.environ.get("KEV_URL")
if EXTERNAL_URL and "/v1/" not in EXTERNAL_URL:
# 允许只给 base(http://host:port),自动补全路径
EXTERNAL_URL = EXTERNAL_URL.rstrip("/") + "/v1/systemone"
def render(v, indent=0):
pad = " " * indent
if v is None:
return ""
if isinstance(v, (str, int, float, bool)):
return str(v)
if isinstance(v, list):
return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v)
return "\n".join(
f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list))
else f"{pad}{k}: {render(x)}" for k, x in v.items())
def option_text(name, desc):
return name if desc is None or desc == "" else f"{name}: {render(desc)}"
def pick(path, max_recs, max_opts, seed=7):
"""抽记录,只留 2..max_opts 个选项且带 label 的题。"""
recs = []
with open(path, encoding="utf-8") as f:
for line in f:
r = json.loads(line)
qs, expect = {}, {}
for qid, q in r["questions"].items():
c = q.get("criteria") or {}
lab = q.get("label")
if lab is None:
continue
if q["type"] == "noul":
k = 2
exp = bool(lab) if isinstance(lab, bool) else str(lab).lower() in ("true", "yes")
elif q["type"] == "score":
k = len(c)
exp = lab if isinstance(lab, int) else None # 整数档位索引
else:
k = len(c)
exp = lab
if not (2 <= k <= max_opts):
continue
if q["type"] == "choice" and lab not in c:
continue
qs[qid] = {"type": q["type"],
"instructions": q["instructions"],
"criteria": c}
expect[qid] = {"kind": q["type"], "answer": exp,
"label": lab, "opts": k}
if qs:
recs.append({"state": r["state"], "questions": qs, "expect": expect})
# 先把**全部**候选读进来再打乱抽取。
# 早先是"读到 max_recs 就 break 然后打乱",结果前 12 条记录恰好全是 noul 题,
# 题型分布完全偏了(还以为是模型问题)。
random.Random(seed).shuffle(recs)
return recs[:max_recs]
def main():
max_opts = int(sys.argv[1]) if len(sys.argv) > 1 else 4
max_recs = int(sys.argv[2]) if len(sys.argv) > 2 else 20
recs = pick(DEV, max_recs, max_opts)
nq = sum(len(r["questions"]) for r in recs)
print(f"取 {len(recs)} 条记录 / {nq} 道题(选项数 ≤{max_opts})", flush=True)
if EXTERNAL_URL:
print(f"使用已运行的服务:{EXTERNAL_URL}", flush=True)
p = None
else:
print("启动服务 …", flush=True)
p = subprocess.Popen([sys.executable, os.path.join(HERE, "kev_serve.py")],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
env={**os.environ, "KEV_PORT": str(PORT)})
t0 = time.time()
if p is not None:
ready = False
while True:
line = p.stdout.readline()
if not line:
break
if b"listening on" in line:
ready = True
break
if time.time() - t0 > 240:
break
if not ready:
print("服务没起来", flush=True)
p.kill()
sys.exit(1)
print(f"服务就绪({time.time()-t0:.1f}s)", flush=True)
stat = {"choice": [0, 0], "noul": [0, 0], "score": [0, 0]}
fails = []
t_all = time.time()
for ri, r in enumerate(recs):
body = json.dumps({"state": r["state"], "questions": r["questions"]},
ensure_ascii=False).encode()
req = urllib.request.Request(EXTERNAL_URL or f"http://127.0.0.1:{PORT}/v1/systemone",
data=body, headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=600) as x:
resp = json.loads(x.read().decode())
ans = resp.get("answers", {})
for qid, ex in r["expect"].items():
got = ans.get(qid)
if not got or "error" in got:
fails.append((ri, qid, "ERR", str(got)[:60]))
continue
k = ex["kind"]
stat[k][1] += 1
if k == "noul":
ok = bool(got["answer"]) == bool(ex["answer"])
elif k == "score":
# label 是整数档位索引 → 与 argmax 档位比(不是与期望等级 score 比)
ok = int(got["argmax"]) == int(ex["label"])
else:
ok = got["answer"] == ex["answer"]
if ok:
stat[k][0] += 1
else:
got_v = got.get("argmax") if k == "score" else got.get("answer")
fails.append((ri, qid, k, f"{got_v} != {ex['label']}"))
if (ri + 1) % 5 == 0:
print(f" {ri+1}/{len(recs)} …", flush=True)
dt = time.time() - t_all
print("\n=== 分题型准确率 ===", flush=True)
tot_o = tot_n = 0
for k, (o, n) in stat.items():
if n:
print(f" {k:7s} {o}/{n} = {o/n:.3f}", flush=True)
tot_o += o
tot_n += n
print(f" 合计 {tot_o}/{tot_n} = {tot_o/max(tot_n,1):.3f}", flush=True)
print(f"\n{nq} 道题共 {dt:.1f}s,平均 {dt/max(nq,1):.2f}s/题", flush=True)
if fails:
print(f"\n失败 {len(fails)} 条,前 10:", flush=True)
for f in fails[:10]:
print(" ", f, flush=True)
# KEV_URL 模式下没有子进程(p is None),不能无条件 kill
if p:
p.kill()
if __name__ == "__main__":
main()