File size: 7,083 Bytes
a846b3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
"""端到端验证在线服务:从 decision-v2 取题 → 走 HTTP → 与标签比。

这一步验的是"服务路径"而不是"批量路径":
  1. kev_serve 里的 render/option_text 与 plan_a 是否一致(不一致准确率会掉)
  2. score 题型能不能跑通(此前只测过 choice/noul)
  3. 三种题型混在一起时的逐题正确率

用法:在 Windows 上运行(服务与 daemon 都在本机)。
    python e2e_check.py [每题最多选项数] [最多记录数]
"""
import json
import os
import random
import subprocess
import sys
import time
import urllib.request

sys.stdout.reconfigure(encoding="utf-8", errors="replace")
HERE = os.path.dirname(os.path.abspath(__file__))
PORT = random.randint(30000, 39999)
# 数据集默认 Windows 本机;在其他机器上跑时用 KEV_DEV 指过去
DEV = os.environ.get("KEV_DEV") or r"D:\kevsrc\evals\decision-v2\development.jsonl"
# 给了 KEV_URL 就直接连**已有服务**,不再自己起进程 ——
# 这样可以从另一台机器(走 SSH 端口转发)验证同一个实例,
# 而不是每次测都新起一个(新起的不代表线上那个的配置)。
EXTERNAL_URL = os.environ.get("KEV_URL")
if EXTERNAL_URL and "/v1/" not in EXTERNAL_URL:
    # 允许只给 base(http://host:port),自动补全路径
    EXTERNAL_URL = EXTERNAL_URL.rstrip("/") + "/v1/systemone"


def render(v, indent=0):
    pad = "  " * indent
    if v is None:
        return ""
    if isinstance(v, (str, int, float, bool)):
        return str(v)
    if isinstance(v, list):
        return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v)
    return "\n".join(
        f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list))
        else f"{pad}{k}: {render(x)}" for k, x in v.items())


def option_text(name, desc):
    return name if desc is None or desc == "" else f"{name}: {render(desc)}"


def pick(path, max_recs, max_opts, seed=7):
    """抽记录,只留 2..max_opts 个选项且带 label 的题。"""
    recs = []
    with open(path, encoding="utf-8") as f:
        for line in f:
            r = json.loads(line)
            qs, expect = {}, {}
            for qid, q in r["questions"].items():
                c = q.get("criteria") or {}
                lab = q.get("label")
                if lab is None:
                    continue
                if q["type"] == "noul":
                    k = 2
                    exp = bool(lab) if isinstance(lab, bool) else str(lab).lower() in ("true", "yes")
                elif q["type"] == "score":
                    k = len(c)
                    exp = lab if isinstance(lab, int) else None   # 整数档位索引
                else:
                    k = len(c)
                    exp = lab
                if not (2 <= k <= max_opts):
                    continue
                if q["type"] == "choice" and lab not in c:
                    continue
                qs[qid] = {"type": q["type"],
                           "instructions": q["instructions"],
                           "criteria": c}
                expect[qid] = {"kind": q["type"], "answer": exp,
                               "label": lab, "opts": k}
            if qs:
                recs.append({"state": r["state"], "questions": qs, "expect": expect})
    # 先把**全部**候选读进来再打乱抽取。
    # 早先是"读到 max_recs 就 break 然后打乱",结果前 12 条记录恰好全是 noul 题,
    # 题型分布完全偏了(还以为是模型问题)。
    random.Random(seed).shuffle(recs)
    return recs[:max_recs]


def main():
    max_opts = int(sys.argv[1]) if len(sys.argv) > 1 else 4
    max_recs = int(sys.argv[2]) if len(sys.argv) > 2 else 20

    recs = pick(DEV, max_recs, max_opts)
    nq = sum(len(r["questions"]) for r in recs)
    print(f"取 {len(recs)} 条记录 / {nq} 道题(选项数 ≤{max_opts})", flush=True)

    if EXTERNAL_URL:
        print(f"使用已运行的服务:{EXTERNAL_URL}", flush=True)
        p = None
    else:
        print("启动服务 …", flush=True)
        p = subprocess.Popen([sys.executable, os.path.join(HERE, "kev_serve.py")],
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             env={**os.environ, "KEV_PORT": str(PORT)})
    t0 = time.time()
    if p is not None:
        ready = False
        while True:
            line = p.stdout.readline()
            if not line:
                break
            if b"listening on" in line:
                ready = True
                break
            if time.time() - t0 > 240:
                break
        if not ready:
            print("服务没起来", flush=True)
            p.kill()
            sys.exit(1)
        print(f"服务就绪({time.time()-t0:.1f}s)", flush=True)

    stat = {"choice": [0, 0], "noul": [0, 0], "score": [0, 0]}
    fails = []
    t_all = time.time()
    for ri, r in enumerate(recs):
        body = json.dumps({"state": r["state"], "questions": r["questions"]},
                          ensure_ascii=False).encode()
        req = urllib.request.Request(EXTERNAL_URL or f"http://127.0.0.1:{PORT}/v1/systemone",
                                     data=body, headers={"Content-Type": "application/json"})
        with urllib.request.urlopen(req, timeout=600) as x:
            resp = json.loads(x.read().decode())
        ans = resp.get("answers", {})
        for qid, ex in r["expect"].items():
            got = ans.get(qid)
            if not got or "error" in got:
                fails.append((ri, qid, "ERR", str(got)[:60]))
                continue
            k = ex["kind"]
            stat[k][1] += 1
            if k == "noul":
                ok = bool(got["answer"]) == bool(ex["answer"])
            elif k == "score":
                # label 是整数档位索引 → 与 argmax 档位比(不是与期望等级 score 比)
                ok = int(got["argmax"]) == int(ex["label"])
            else:
                ok = got["answer"] == ex["answer"]
            if ok:
                stat[k][0] += 1
            else:
                got_v = got.get("argmax") if k == "score" else got.get("answer")
                fails.append((ri, qid, k, f"{got_v} != {ex['label']}"))
        if (ri + 1) % 5 == 0:
            print(f"  {ri+1}/{len(recs)} …", flush=True)

    dt = time.time() - t_all
    print("\n=== 分题型准确率 ===", flush=True)
    tot_o = tot_n = 0
    for k, (o, n) in stat.items():
        if n:
            print(f"  {k:7s} {o}/{n} = {o/n:.3f}", flush=True)
        tot_o += o
        tot_n += n
    print(f"  合计    {tot_o}/{tot_n} = {tot_o/max(tot_n,1):.3f}", flush=True)
    print(f"\n{nq} 道题共 {dt:.1f}s,平均 {dt/max(nq,1):.2f}s/题", flush=True)
    if fails:
        print(f"\n失败 {len(fails)} 条,前 10:", flush=True)
        for f in fails[:10]:
            print("  ", f, flush=True)
    # KEV_URL 模式下没有子进程(p is None),不能无条件 kill
    if p:
        p.kill()


if __name__ == "__main__":
    main()