"""端到端验证在线服务:从 decision-v2 取题 → 走 HTTP → 与标签比。 这一步验的是"服务路径"而不是"批量路径": 1. kev_serve 里的 render/option_text 与 plan_a 是否一致(不一致准确率会掉) 2. score 题型能不能跑通(此前只测过 choice/noul) 3. 三种题型混在一起时的逐题正确率 用法:在 Windows 上运行(服务与 daemon 都在本机)。 python e2e_check.py [每题最多选项数] [最多记录数] """ import json import os import random import subprocess import sys import time import urllib.request sys.stdout.reconfigure(encoding="utf-8", errors="replace") HERE = os.path.dirname(os.path.abspath(__file__)) PORT = random.randint(30000, 39999) # 数据集默认 Windows 本机;在其他机器上跑时用 KEV_DEV 指过去 DEV = os.environ.get("KEV_DEV") or r"D:\kevsrc\evals\decision-v2\development.jsonl" # 给了 KEV_URL 就直接连**已有服务**,不再自己起进程 —— # 这样可以从另一台机器(走 SSH 端口转发)验证同一个实例, # 而不是每次测都新起一个(新起的不代表线上那个的配置)。 EXTERNAL_URL = os.environ.get("KEV_URL") if EXTERNAL_URL and "/v1/" not in EXTERNAL_URL: # 允许只给 base(http://host:port),自动补全路径 EXTERNAL_URL = EXTERNAL_URL.rstrip("/") + "/v1/systemone" def render(v, indent=0): pad = " " * indent if v is None: return "" if isinstance(v, (str, int, float, bool)): return str(v) if isinstance(v, list): return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v) return "\n".join( f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list)) else f"{pad}{k}: {render(x)}" for k, x in v.items()) def option_text(name, desc): return name if desc is None or desc == "" else f"{name}: {render(desc)}" def pick(path, max_recs, max_opts, seed=7): """抽记录,只留 2..max_opts 个选项且带 label 的题。""" recs = [] with open(path, encoding="utf-8") as f: for line in f: r = json.loads(line) qs, expect = {}, {} for qid, q in r["questions"].items(): c = q.get("criteria") or {} lab = q.get("label") if lab is None: continue if q["type"] == "noul": k = 2 exp = bool(lab) if isinstance(lab, bool) else str(lab).lower() in ("true", "yes") elif q["type"] == "score": k = len(c) exp = lab if isinstance(lab, int) else None # 整数档位索引 else: k = len(c) exp = lab if not (2 <= k <= max_opts): continue if q["type"] == "choice" and lab not in c: continue qs[qid] = {"type": q["type"], "instructions": q["instructions"], "criteria": c} expect[qid] = {"kind": q["type"], "answer": exp, "label": lab, "opts": k} if qs: recs.append({"state": r["state"], "questions": qs, "expect": expect}) # 先把**全部**候选读进来再打乱抽取。 # 早先是"读到 max_recs 就 break 然后打乱",结果前 12 条记录恰好全是 noul 题, # 题型分布完全偏了(还以为是模型问题)。 random.Random(seed).shuffle(recs) return recs[:max_recs] def main(): max_opts = int(sys.argv[1]) if len(sys.argv) > 1 else 4 max_recs = int(sys.argv[2]) if len(sys.argv) > 2 else 20 recs = pick(DEV, max_recs, max_opts) nq = sum(len(r["questions"]) for r in recs) print(f"取 {len(recs)} 条记录 / {nq} 道题(选项数 ≤{max_opts})", flush=True) if EXTERNAL_URL: print(f"使用已运行的服务:{EXTERNAL_URL}", flush=True) p = None else: print("启动服务 …", flush=True) p = subprocess.Popen([sys.executable, os.path.join(HERE, "kev_serve.py")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env={**os.environ, "KEV_PORT": str(PORT)}) t0 = time.time() if p is not None: ready = False while True: line = p.stdout.readline() if not line: break if b"listening on" in line: ready = True break if time.time() - t0 > 240: break if not ready: print("服务没起来", flush=True) p.kill() sys.exit(1) print(f"服务就绪({time.time()-t0:.1f}s)", flush=True) stat = {"choice": [0, 0], "noul": [0, 0], "score": [0, 0]} fails = [] t_all = time.time() for ri, r in enumerate(recs): body = json.dumps({"state": r["state"], "questions": r["questions"]}, ensure_ascii=False).encode() req = urllib.request.Request(EXTERNAL_URL or f"http://127.0.0.1:{PORT}/v1/systemone", data=body, headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=600) as x: resp = json.loads(x.read().decode()) ans = resp.get("answers", {}) for qid, ex in r["expect"].items(): got = ans.get(qid) if not got or "error" in got: fails.append((ri, qid, "ERR", str(got)[:60])) continue k = ex["kind"] stat[k][1] += 1 if k == "noul": ok = bool(got["answer"]) == bool(ex["answer"]) elif k == "score": # label 是整数档位索引 → 与 argmax 档位比(不是与期望等级 score 比) ok = int(got["argmax"]) == int(ex["label"]) else: ok = got["answer"] == ex["answer"] if ok: stat[k][0] += 1 else: got_v = got.get("argmax") if k == "score" else got.get("answer") fails.append((ri, qid, k, f"{got_v} != {ex['label']}")) if (ri + 1) % 5 == 0: print(f" {ri+1}/{len(recs)} …", flush=True) dt = time.time() - t_all print("\n=== 分题型准确率 ===", flush=True) tot_o = tot_n = 0 for k, (o, n) in stat.items(): if n: print(f" {k:7s} {o}/{n} = {o/n:.3f}", flush=True) tot_o += o tot_n += n print(f" 合计 {tot_o}/{tot_n} = {tot_o/max(tot_n,1):.3f}", flush=True) print(f"\n{nq} 道题共 {dt:.1f}s,平均 {dt/max(nq,1):.2f}s/题", flush=True) if fails: print(f"\n失败 {len(fails)} 条,前 10:", flush=True) for f in fails[:10]: print(" ", f, flush=True) # KEV_URL 模式下没有子进程(p is None),不能无条件 kill if p: p.kill() if __name__ == "__main__": main()