File size: 18,558 Bytes
a846b3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
"""Kev 在 RK1828 上的在线决策服务(跑在 Windows)。

架构:
    HTTP (本文件)  --stdin/stdout-->  daemon.exe  --NPU-->  logits
                       host 侧 numpy 做 反解 + 指针打分

一次请求 = K+1 次 prefill(<decide> 一次 + 每个 </opt> 一次),
因为图的输出只保留最后一个位置,取别的位置只能把序列截断重跑。

接口:POST /v1/systemone
  {"state": <任意 JSON>, "questions": {"q1": {"type":"choice|noul|score",
   "instructions": "...", "criteria": {...} | [...]}}}
"""
import base64
import json
import os
import subprocess
import sys
import threading
import time
import traceback
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

import numpy as np

# 所有路径都从本文件位置推导,不写死盘符 —— 交付后整个目录搬到任何地方都能用。
# 布局:<包根>/kevrunner/kev_serve.py,<包根>/models/<模型>,<包根>/runtime/rknn3_110
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
# 两处可用环境变量覆盖,方便放进别的目录布局(不设时行为与原先完全一致)
MODEL = os.environ.get("KEV_MODEL_DIR") or os.path.join(ROOT, "models", "Kev-0.8B")
SEP = os.sep
WORK = HERE
DAEMON = os.environ.get("KEV_DAEMON") or os.path.join(
    WORK, r"rknn3_session_test_demo\daemon.exe")
CUR = os.path.join(WORK, r"out\cur.bin")
# plan task has no console; pipe stdout EOFs immediately there, so use a file channel
CHAN = os.path.join(WORK, "chan.log")

# 官方 kev/model.py::SPECIAL 的 id
T_STATE, T_Q, T_OPT, T_OPT_END, T_DECIDE = 248060, 248061, 248049, 248050, 248062
VOCAB, HID = 248320, 1024
MAX_OPTS = 255


# ---------- 渲染(对齐官方 kev/api.py) ----------
def render(v, indent=0):
    pad = "  " * indent
    if v is None:
        return ""
    if isinstance(v, (str, int, float, bool)):
        return str(v)
    if isinstance(v, list):
        return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v)
    return "\n".join(
        f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list))
        else f"{pad}{k}: {render(x)}" for k, x in v.items())


def option_text(name, desc):
    return name if desc is None or desc == "" else f"{name}: {render(desc)}"


def question_options(q):
    """返回 (选项文本列表, 选项名列表, 题型)。"""
    t = q["type"]
    if t == "noul":
        return ["No", "Yes"], ["false", "true"], "noul"
    if t == "score":
        crit = q["criteria"]
        return [render(c) for c in crit], [str(i) for i in range(len(crit))], "score"
    crit = q["criteria"]
    names = list(crit)
    return [option_text(k, crit[k]) for k in names], names, "choice"


def round_prob(x):
    return round(float(x), 4)


def choice_confidence(p):
    """(p_max - 1/K) / (1 - 1/K):均匀分布为 0,确定时为 1。"""
    K = len(p)
    return 1.0 if K == 1 else (float(max(p)) - 1 / K) / (1 - 1 / K)


def score_confidence(p):
    """1 - E|level - mode| / D,D = 均匀分布的平均绝对偏差。"""
    L = len(p)
    if L == 1:
        return 1.0
    p = [float(x) for x in p]
    mode = max(range(L), key=p.__getitem__)
    D = sum(abs(i - (L - 1) / 2) for i in range(L)) / L
    return max(0.0, 1.0 - sum(pi * abs(i - mode) for i, pi in enumerate(p)) / D)


# ---------- daemon 通信 ----------
class Daemon:
    def __init__(self):
        self.p = subprocess.Popen(
            # 用 ntpath 拼:本文件在 Windows 上跑,os.path.join 会按 Windows 规则走,
            # 但显式用 "\\" 更稳(曾被混合分隔符坑过)
            [DAEMON]
            + [MODEL + SEP + f"Kev-0.8B{ext}"
               for ext in (".rknn", ".weight", ".tokenizer.gguf", ".embed.bin")]
            + ["1024", "1", "0xff", "0", "0", "key.env"],
            cwd=os.path.dirname(DAEMON),
            stdin=subprocess.PIPE,
            # 计划任务没有控制台,daemon 接管道时 stdout 会立刻 EOF
            # (rc=None 但进程还活着)。落文件,有无控制台都成立。
            stdout=open(CHAN, "w+b", buffering=0),
            stderr=subprocess.STDOUT)
        self.lock = threading.Lock()
        self._buf = b""
        self.offset = 0
        self._buf = b""
        # 等 daemon 在通道文件里写出 READY_DAEMON(模型加载约 30 s,给 180 s)
        if not self._wait_for("READY_DAEMON", 180):
            raise RuntimeError(
                f"daemon 180s 内没有 READY_DAEMON(rc={self.p.poll()}),"
                f"最后输出:{self._tail() or '(空)'}")

    def _tail(self, n=6):
        try:
            with open(CHAN, "r", encoding="utf-8", errors="replace") as f:
                return " | ".join(l.rstrip()[:110] for l in f.readlines()[-n:])
        except OSError:
            return ""

    def _wait_for(self, prefix, timeout):
        """轮询通道文件,从 self.offset 往后找以 prefix 开头的行。

        用文件而不是管道:计划任务没有控制台,管道方案在那里会立刻 EOF。
        """
        t0 = time.time()
        while time.time() - t0 < timeout:
            chunk = b""
            try:
                with open(CHAN, "rb") as f:
                    f.seek(self.offset)
                    chunk = f.read()
            except OSError:
                pass
            if chunk:
                self.offset += len(chunk)
                buf = self._buf + chunk
                parts = buf.split(b"\n")
                self._buf = parts.pop()          # 不完整的尾巴留到下次
                for raw in parts:
                    ln = raw.decode("utf-8", "replace").rstrip()
                    if ln.startswith(prefix):
                        return ln
            if self.p.poll() is not None:
                return None
            time.sleep(0.05)
        return None

    def _cmd(self, line, stop_prefix):
        self.p.stdin.write((line + "\n").encode("utf-8"))
        self.p.stdin.flush()
        out = self._wait_for(stop_prefix, 180)
        if out is None:
            raise RuntimeError(
                f"等 {stop_prefix} 超时/退出(rc={self.p.poll()}):{self._tail()}")
        return out

    def tok(self, text):
        b64 = base64.b64encode(text.encode("utf-8")).decode("ascii")
        with self.lock:
            out = self._cmd("TOK " + b64, "IDS ")
        parts = out.split()
        n = int(parts[1])
        return [int(x) for x in parts[2:2 + n]]

    def prefill(self, ids, append=False):
        """跑一次 prefill,返回 248320 维 logits。

        append=False → R(先清 kvcache,从头算)
        append=True  → A(接在上一次之后,只算新增这段)
        实测两者余弦 ≥ 0.998,top-1 一致,所以同一题内可以安全用增量。
        """
        with self.lock:
            self._cmd(("A " if append else "R ") + " ".join(str(x) for x in ids), "DONE")
            return np.fromfile(CUR, dtype=np.float16).astype(np.float32)

    def close(self):
        try:
            self.p.stdin.write(b"quit\n")
            self.p.stdin.flush()
        except Exception:
            pass
        try:
            self.p.wait(timeout=10)
        except Exception:
            self.p.kill()


# ---------- 打分层 ----------
class Scorer:
    def __init__(self, assets):
        self.Ginv = np.load(os.path.join(assets, "Ginv.npy"))          # 1024x1024
        self.q = np.load(os.path.join(assets, "head_q.npy"))           # 256x1024
        self.k = np.load(os.path.join(assets, "head_k.npy"))           # 256x1024
        self.T = float(np.load(os.path.join(assets, "temperature.npy"))[0])
        # 一次性读进内存(500 MB fp16)。用 memmap 的话每次 hidden() 都要从磁盘
        # 重新读一遍 500 MB —— 实测把单请求从 7.6 s 拖到很慢。
        print("loading embed weights to RAM …", flush=True)
        W16 = np.fromfile(os.path.join(MODEL, "Kev-0.8B.embed.bin"),
                          dtype=np.float16).reshape(VOCAB, HID)
        # 预转 float32 常驻(1 GB)。否则每次 hidden() 都要转 2.68 亿个元素,
        # 实测单请求 7.6 s 里大部分耗在这上面。
        print("casting weights to fp32 …", flush=True)
        self.W = np.ascontiguousarray(W16, dtype=np.float32)
        del W16

    def hidden(self, logits):
        """h = (WᵀW)⁻¹ Wᵀ logits"""
        # Wᵀ @ logits:memmap 分块算,避免一次性 500 MB 上浮
        rhs = np.zeros(HID, dtype=np.float32)
        step = 16384
        for s in range(0, VOCAB, step):
            e = min(s + step, VOCAB)
            rhs += self.W[s:e].T @ logits[s:e]
        return self.Ginv @ rhs

    def scores(self, h_decide, h_opts):
        z = (h_opts @ self.k.T) @ (self.q @ h_decide) / np.sqrt(self.q.shape[0])
        z = z / self.T
        z = z - z.max()
        p = np.exp(z)
        return p / p.sum()


# ---------- 决策 ----------
class BadRequest(Exception):
    """客户端参数错误 → 400。和「服务出故障」必须分开:
    参数错回 500 会让人以为服务挂了,日志里也全是假异常。"""


def validate_request(req):
    """把能判的错都在**进模型之前**判掉。

    这些情况以前会一路走到 question_options 里抛 KeyError,
    被外层 catch 成 500 并回传 traceback —— 既误导调用方,又泄漏实现细节。
    还有两处更糟的:缺 state 会白跑一次 prefill(几百毫秒),
    未知 type 会被静默当成 choice 处理(拼错 "choices" 也能拿到答案)。
    """
    if not isinstance(req, dict):
        raise BadRequest("body must be a JSON object")

    state = req.get("state")
    # state 允许三种形态,这是 decision-v2 的真实分布,不是想当然:
    #   str  664 条 / dict 440 条 / list 72 条
    # 一开始只允许 str,结果把 512 条合法请求挡成 400 —— 校验过严比不校验更糟。
    if state is None:
        raise BadRequest("state is required")
    if isinstance(state, str):
        if not state.strip():
            raise BadRequest("state must not be empty")
    elif isinstance(state, (dict, list)):
        if not state:
            raise BadRequest("state must not be empty")
    else:
        raise BadRequest(
            f"state must be a string, object or array, got {type(state).__name__}")

    qs = req.get("questions")
    if not isinstance(qs, dict):
        raise BadRequest("questions is required: an object {id: question}")
    if not qs:
        # 空 questions 以前返回 {"answers": {}} —— 拼错字段名(比如写成 question)
        # 会静默"成功",调用方以为模型没意见。宁可报错。
        raise BadRequest("questions must not be empty")

    for qid, q in qs.items():
        if not isinstance(q, dict):
            raise BadRequest(f"questions[{qid}] must be an object")
        kind = q.get("type")
        if kind not in ("noul", "choice", "score"):
            raise BadRequest(
                f"questions[{qid}].type must be one of noul/choice/score, got {kind!r}")
        if kind == "choice":
            crit = q.get("criteria")
            if not isinstance(crit, dict) or not crit:
                raise BadRequest(f"questions[{qid}].criteria must be a non-empty object")
        elif kind == "score":
            crit = q.get("criteria")
            if not isinstance(crit, list) or not crit:
                raise BadRequest(f"questions[{qid}].criteria must be a non-empty array")


# 一次 decide 会连续调用多次 prefill(K+1 次),中间必须独占 daemon。
# 只在单次 _cmd 里加锁是不够的 —— HTTP 层是多线程的,两个请求会交替
# 使用同一份 kvcache,后一个的 append 会接在前一个的序列后面,结果全乱。
DECIDE_LOCK = threading.Lock()

MAX_BODY = 1 << 20   # 1 MB,够放很长的 state,也挡住恶意大包


def decide(dmn, sc, state, questions):
    st_ids = dmn.tok(render(state))
    S = [T_STATE] + st_ids
    out = {}
    for qid, q in questions.items():
        texts, names, kind = question_options(q)
        K = len(texts)
        if not 1 <= K <= MAX_OPTS:
            out[qid] = {"error": f"options must be 1..{MAX_OPTS}"}
            continue
        ins = [T_Q] + dmn.tok(render(q.get("instructions") or ""))
        spans = [[T_OPT] + dmn.tok(t) + [T_OPT_END] for t in texts]
        ids = S + ins + [x for sp in spans for x in sp] + [T_DECIDE]

        dec = len(ids) - 1
        ends, cur = [], len(S) + len(ins)
        for sp in spans:
            ends.append(cur + len(sp) - 1)
            cur += len(sp)

        # 增量 prefill:只算一次全序列,其余位置靠逐段追加。
        # 全量要算 sum(len)=约 K+1 倍的 token;增量只算整条序列一遍。
        prev, h_opts = -1, []
        for i, e in enumerate(ends):
            h_opts.append(sc.hidden(dmn.prefill(ids[prev + 1:e + 1], append=(i > 0))))
            prev = e
        h_dec = sc.hidden(dmn.prefill(ids[prev + 1:dec + 1], append=True))
        h_opts = np.stack(h_opts)
        prob = sc.scores(h_dec, h_opts)

        # 输出结构对齐官方 kev/api.py::to_answers
        if kind == "noul":
            out[qid] = {"type": "noul", "noul": round_prob(prob[1]),
                        "answer": bool(prob[1] >= 0.5)}
        elif kind == "score":
            lv = sum(i * pi for i, pi in enumerate(prob))
            out[qid] = {"type": "score", "score": round_prob(lv),
                        "legend": texts,
                        "probabilities": {str(i): round_prob(v) for i, v in enumerate(prob)},
                        "confidence": round_prob(score_confidence(prob)),
                        "argmax": int(prob.argmax())}
        else:
            i = int(prob.argmax())
            out[qid] = {"type": "choice", "answer": names[i],
                        "confidence": round_prob(choice_confidence(prob)),
                        "probabilities": {n: round_prob(p) for n, p in zip(names, prob)}}
    return {"answers": out}


def main():
    assets = os.path.join(HERE, "serve_assets")
    print("loading assets …", flush=True)
    sc = Scorer(assets)
    print("starting daemon …", flush=True)
    # 计划任务不继承 PATH。daemon.exe 依赖 runtime\lib 里的 librknn3_api.dll 等,
    # 找不到会以 0xC0000135 (rc=3221225781) 直接退出、一个字都不输出。
    rt = os.path.join(ROOT, "runtime", "rknn3_110")
    for d in (os.path.join(rt, "lib"), os.path.join(rt, "bin")):
        if os.path.isdir(d) and d not in os.environ.get("PATH", ""):
            os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
    print(f"[env] PATH head = {os.environ['PATH'].split(os.pathsep)[:2]}", flush=True)
    dmn = Daemon()
    print("daemon ready", flush=True)

    class H(BaseHTTPRequestHandler):
        def _json(self, code, obj):
            raw = json.dumps(obj, ensure_ascii=False).encode()
            self.send_response(code)
            self.send_header("Content-Type", "application/json; charset=utf-8")
            self.send_header("Content-Length", str(len(raw)))
            self.end_headers()
            self.wfile.write(raw)

        def do_POST(self):
            if self.path != "/v1/systemone":
                self.send_error(404)
                return

            n = int(self.headers.get("Content-Length", 0) or 0)
            if n > MAX_BODY:
                self._json(413, {"error": f"body too large: {n} bytes (limit {MAX_BODY})"})
                return
            raw_in = self.rfile.read(n) if n > 0 else b""
            if not raw_in:
                self._json(400, {"error": "empty body"})
                return

            try:
                req = json.loads(raw_in)
            except json.JSONDecodeError as e:
                # 4xx 而不是 5xx:这是调用方的问题,回 500 会让人以为服务挂了
                self._json(400, {"error": f"invalid JSON at line {e.lineno} col {e.colno}: {e.msg}"})
                return

            try:
                validate_request(req)
            except BadRequest as e:
                self._json(400, {"error": str(e)})
                return

            try:
                # 整个决策独占 daemon:中间的 K+1 次 prefill 不能被别的请求插进来
                with DECIDE_LOCK:
                    body = decide(dmn, sc, req["state"], req["questions"])
                self._json(200, body)
            except Exception as e:
                # traceback 只进日志。响应里给调用方也没用,还泄漏实现细节。
                with open(os.path.join(HERE, "serve_err.log"), "a", encoding="utf-8") as f:
                    f.write(traceback.format_exc() + "\n")
                self._json(500, {"error": f"internal error: {type(e).__name__}: {e}"})

        def do_GET(self):
            # 探活:计划任务起来后用它确认"服务真的能答"
            if self.path == "/healthz":
                raw = json.dumps({"ok": True, "model": "Kev-0.8B",
                                  "pid": os.getpid()}).encode()
                self.send_response(200)
                self.send_header("Content-Type", "application/json")
                self.send_header("Content-Length", str(len(raw)))
                self.end_headers()
                self.wfile.write(raw)
            elif self.path == "/v1/systemone":
                # 存在但不接受 GET:405 比 404 准确,调用方一眼看出是方法错了
                self.send_response(405)
                self.send_header("Allow", "POST")
                self.send_header("Content-Type", "application/json; charset=utf-8")
                raw = json.dumps({"error": "use POST"}).encode()
                self.send_header("Content-Length", str(len(raw)))
                self.end_headers()
                self.wfile.write(raw)
            else:
                self.send_error(404)

        def log_message(self, *a):
            pass

    port = int(os.environ.get("KEV_PORT", "17890"))
    host = os.environ.get("KEV_HOST", "127.0.0.1")
    try:
        srv = ThreadingHTTPServer((host, port), H)
    except OSError as e:
        # 端口被占是最常见的启动失败原因(旧进程没退干净),必须说清楚
        print(f"BIND_FAILED {host}:{port} -> {e}", flush=True)
        raise SystemExit(2)
    print(f"listening on {host}:{port}", flush=True)
    srv.serve_forever()


if __name__ == "__main__":
    main()