"""Kev 在 RK1828 上的在线决策服务(跑在 Windows)。 架构: HTTP (本文件) --stdin/stdout--> daemon.exe --NPU--> logits host 侧 numpy 做 反解 + 指针打分 一次请求 = K+1 次 prefill( 一次 + 每个 一次), 因为图的输出只保留最后一个位置,取别的位置只能把序列截断重跑。 接口:POST /v1/systemone {"state": <任意 JSON>, "questions": {"q1": {"type":"choice|noul|score", "instructions": "...", "criteria": {...} | [...]}}} """ import base64 import json import os import subprocess import sys import threading import time import traceback from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import numpy as np # 所有路径都从本文件位置推导,不写死盘符 —— 交付后整个目录搬到任何地方都能用。 # 布局:<包根>/kevrunner/kev_serve.py,<包根>/models/<模型>,<包根>/runtime/rknn3_110 HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) # 两处可用环境变量覆盖,方便放进别的目录布局(不设时行为与原先完全一致) MODEL = os.environ.get("KEV_MODEL_DIR") or os.path.join(ROOT, "models", "Kev-0.8B") SEP = os.sep WORK = HERE DAEMON = os.environ.get("KEV_DAEMON") or os.path.join( WORK, r"rknn3_session_test_demo\daemon.exe") CUR = os.path.join(WORK, r"out\cur.bin") # plan task has no console; pipe stdout EOFs immediately there, so use a file channel CHAN = os.path.join(WORK, "chan.log") # 官方 kev/model.py::SPECIAL 的 id T_STATE, T_Q, T_OPT, T_OPT_END, T_DECIDE = 248060, 248061, 248049, 248050, 248062 VOCAB, HID = 248320, 1024 MAX_OPTS = 255 # ---------- 渲染(对齐官方 kev/api.py) ---------- def render(v, indent=0): pad = " " * indent if v is None: return "" if isinstance(v, (str, int, float, bool)): return str(v) if isinstance(v, list): return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v) return "\n".join( f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list)) else f"{pad}{k}: {render(x)}" for k, x in v.items()) def option_text(name, desc): return name if desc is None or desc == "" else f"{name}: {render(desc)}" def question_options(q): """返回 (选项文本列表, 选项名列表, 题型)。""" t = q["type"] if t == "noul": return ["No", "Yes"], ["false", "true"], "noul" if t == "score": crit = q["criteria"] return [render(c) for c in crit], [str(i) for i in range(len(crit))], "score" crit = q["criteria"] names = list(crit) return [option_text(k, crit[k]) for k in names], names, "choice" def round_prob(x): return round(float(x), 4) def choice_confidence(p): """(p_max - 1/K) / (1 - 1/K):均匀分布为 0,确定时为 1。""" K = len(p) return 1.0 if K == 1 else (float(max(p)) - 1 / K) / (1 - 1 / K) def score_confidence(p): """1 - E|level - mode| / D,D = 均匀分布的平均绝对偏差。""" L = len(p) if L == 1: return 1.0 p = [float(x) for x in p] mode = max(range(L), key=p.__getitem__) D = sum(abs(i - (L - 1) / 2) for i in range(L)) / L return max(0.0, 1.0 - sum(pi * abs(i - mode) for i, pi in enumerate(p)) / D) # ---------- daemon 通信 ---------- class Daemon: def __init__(self): self.p = subprocess.Popen( # 用 ntpath 拼:本文件在 Windows 上跑,os.path.join 会按 Windows 规则走, # 但显式用 "\\" 更稳(曾被混合分隔符坑过) [DAEMON] + [MODEL + SEP + f"Kev-0.8B{ext}" for ext in (".rknn", ".weight", ".tokenizer.gguf", ".embed.bin")] + ["1024", "1", "0xff", "0", "0", "key.env"], cwd=os.path.dirname(DAEMON), stdin=subprocess.PIPE, # 计划任务没有控制台,daemon 接管道时 stdout 会立刻 EOF # (rc=None 但进程还活着)。落文件,有无控制台都成立。 stdout=open(CHAN, "w+b", buffering=0), stderr=subprocess.STDOUT) self.lock = threading.Lock() self._buf = b"" self.offset = 0 self._buf = b"" # 等 daemon 在通道文件里写出 READY_DAEMON(模型加载约 30 s,给 180 s) if not self._wait_for("READY_DAEMON", 180): raise RuntimeError( f"daemon 180s 内没有 READY_DAEMON(rc={self.p.poll()})," f"最后输出:{self._tail() or '(空)'}") def _tail(self, n=6): try: with open(CHAN, "r", encoding="utf-8", errors="replace") as f: return " | ".join(l.rstrip()[:110] for l in f.readlines()[-n:]) except OSError: return "" def _wait_for(self, prefix, timeout): """轮询通道文件,从 self.offset 往后找以 prefix 开头的行。 用文件而不是管道:计划任务没有控制台,管道方案在那里会立刻 EOF。 """ t0 = time.time() while time.time() - t0 < timeout: chunk = b"" try: with open(CHAN, "rb") as f: f.seek(self.offset) chunk = f.read() except OSError: pass if chunk: self.offset += len(chunk) buf = self._buf + chunk parts = buf.split(b"\n") self._buf = parts.pop() # 不完整的尾巴留到下次 for raw in parts: ln = raw.decode("utf-8", "replace").rstrip() if ln.startswith(prefix): return ln if self.p.poll() is not None: return None time.sleep(0.05) return None def _cmd(self, line, stop_prefix): self.p.stdin.write((line + "\n").encode("utf-8")) self.p.stdin.flush() out = self._wait_for(stop_prefix, 180) if out is None: raise RuntimeError( f"等 {stop_prefix} 超时/退出(rc={self.p.poll()}):{self._tail()}") return out def tok(self, text): b64 = base64.b64encode(text.encode("utf-8")).decode("ascii") with self.lock: out = self._cmd("TOK " + b64, "IDS ") parts = out.split() n = int(parts[1]) return [int(x) for x in parts[2:2 + n]] def prefill(self, ids, append=False): """跑一次 prefill,返回 248320 维 logits。 append=False → R(先清 kvcache,从头算) append=True → A(接在上一次之后,只算新增这段) 实测两者余弦 ≥ 0.998,top-1 一致,所以同一题内可以安全用增量。 """ with self.lock: self._cmd(("A " if append else "R ") + " ".join(str(x) for x in ids), "DONE") return np.fromfile(CUR, dtype=np.float16).astype(np.float32) def close(self): try: self.p.stdin.write(b"quit\n") self.p.stdin.flush() except Exception: pass try: self.p.wait(timeout=10) except Exception: self.p.kill() # ---------- 打分层 ---------- class Scorer: def __init__(self, assets): self.Ginv = np.load(os.path.join(assets, "Ginv.npy")) # 1024x1024 self.q = np.load(os.path.join(assets, "head_q.npy")) # 256x1024 self.k = np.load(os.path.join(assets, "head_k.npy")) # 256x1024 self.T = float(np.load(os.path.join(assets, "temperature.npy"))[0]) # 一次性读进内存(500 MB fp16)。用 memmap 的话每次 hidden() 都要从磁盘 # 重新读一遍 500 MB —— 实测把单请求从 7.6 s 拖到很慢。 print("loading embed weights to RAM …", flush=True) W16 = np.fromfile(os.path.join(MODEL, "Kev-0.8B.embed.bin"), dtype=np.float16).reshape(VOCAB, HID) # 预转 float32 常驻(1 GB)。否则每次 hidden() 都要转 2.68 亿个元素, # 实测单请求 7.6 s 里大部分耗在这上面。 print("casting weights to fp32 …", flush=True) self.W = np.ascontiguousarray(W16, dtype=np.float32) del W16 def hidden(self, logits): """h = (WᵀW)⁻¹ Wᵀ logits""" # Wᵀ @ logits:memmap 分块算,避免一次性 500 MB 上浮 rhs = np.zeros(HID, dtype=np.float32) step = 16384 for s in range(0, VOCAB, step): e = min(s + step, VOCAB) rhs += self.W[s:e].T @ logits[s:e] return self.Ginv @ rhs def scores(self, h_decide, h_opts): z = (h_opts @ self.k.T) @ (self.q @ h_decide) / np.sqrt(self.q.shape[0]) z = z / self.T z = z - z.max() p = np.exp(z) return p / p.sum() # ---------- 决策 ---------- class BadRequest(Exception): """客户端参数错误 → 400。和「服务出故障」必须分开: 参数错回 500 会让人以为服务挂了,日志里也全是假异常。""" def validate_request(req): """把能判的错都在**进模型之前**判掉。 这些情况以前会一路走到 question_options 里抛 KeyError, 被外层 catch 成 500 并回传 traceback —— 既误导调用方,又泄漏实现细节。 还有两处更糟的:缺 state 会白跑一次 prefill(几百毫秒), 未知 type 会被静默当成 choice 处理(拼错 "choices" 也能拿到答案)。 """ if not isinstance(req, dict): raise BadRequest("body must be a JSON object") state = req.get("state") # state 允许三种形态,这是 decision-v2 的真实分布,不是想当然: # str 664 条 / dict 440 条 / list 72 条 # 一开始只允许 str,结果把 512 条合法请求挡成 400 —— 校验过严比不校验更糟。 if state is None: raise BadRequest("state is required") if isinstance(state, str): if not state.strip(): raise BadRequest("state must not be empty") elif isinstance(state, (dict, list)): if not state: raise BadRequest("state must not be empty") else: raise BadRequest( f"state must be a string, object or array, got {type(state).__name__}") qs = req.get("questions") if not isinstance(qs, dict): raise BadRequest("questions is required: an object {id: question}") if not qs: # 空 questions 以前返回 {"answers": {}} —— 拼错字段名(比如写成 question) # 会静默"成功",调用方以为模型没意见。宁可报错。 raise BadRequest("questions must not be empty") for qid, q in qs.items(): if not isinstance(q, dict): raise BadRequest(f"questions[{qid}] must be an object") kind = q.get("type") if kind not in ("noul", "choice", "score"): raise BadRequest( f"questions[{qid}].type must be one of noul/choice/score, got {kind!r}") if kind == "choice": crit = q.get("criteria") if not isinstance(crit, dict) or not crit: raise BadRequest(f"questions[{qid}].criteria must be a non-empty object") elif kind == "score": crit = q.get("criteria") if not isinstance(crit, list) or not crit: raise BadRequest(f"questions[{qid}].criteria must be a non-empty array") # 一次 decide 会连续调用多次 prefill(K+1 次),中间必须独占 daemon。 # 只在单次 _cmd 里加锁是不够的 —— HTTP 层是多线程的,两个请求会交替 # 使用同一份 kvcache,后一个的 append 会接在前一个的序列后面,结果全乱。 DECIDE_LOCK = threading.Lock() MAX_BODY = 1 << 20 # 1 MB,够放很长的 state,也挡住恶意大包 def decide(dmn, sc, state, questions): st_ids = dmn.tok(render(state)) S = [T_STATE] + st_ids out = {} for qid, q in questions.items(): texts, names, kind = question_options(q) K = len(texts) if not 1 <= K <= MAX_OPTS: out[qid] = {"error": f"options must be 1..{MAX_OPTS}"} continue ins = [T_Q] + dmn.tok(render(q.get("instructions") or "")) spans = [[T_OPT] + dmn.tok(t) + [T_OPT_END] for t in texts] ids = S + ins + [x for sp in spans for x in sp] + [T_DECIDE] dec = len(ids) - 1 ends, cur = [], len(S) + len(ins) for sp in spans: ends.append(cur + len(sp) - 1) cur += len(sp) # 增量 prefill:只算一次全序列,其余位置靠逐段追加。 # 全量要算 sum(len)=约 K+1 倍的 token;增量只算整条序列一遍。 prev, h_opts = -1, [] for i, e in enumerate(ends): h_opts.append(sc.hidden(dmn.prefill(ids[prev + 1:e + 1], append=(i > 0)))) prev = e h_dec = sc.hidden(dmn.prefill(ids[prev + 1:dec + 1], append=True)) h_opts = np.stack(h_opts) prob = sc.scores(h_dec, h_opts) # 输出结构对齐官方 kev/api.py::to_answers if kind == "noul": out[qid] = {"type": "noul", "noul": round_prob(prob[1]), "answer": bool(prob[1] >= 0.5)} elif kind == "score": lv = sum(i * pi for i, pi in enumerate(prob)) out[qid] = {"type": "score", "score": round_prob(lv), "legend": texts, "probabilities": {str(i): round_prob(v) for i, v in enumerate(prob)}, "confidence": round_prob(score_confidence(prob)), "argmax": int(prob.argmax())} else: i = int(prob.argmax()) out[qid] = {"type": "choice", "answer": names[i], "confidence": round_prob(choice_confidence(prob)), "probabilities": {n: round_prob(p) for n, p in zip(names, prob)}} return {"answers": out} def main(): assets = os.path.join(HERE, "serve_assets") print("loading assets …", flush=True) sc = Scorer(assets) print("starting daemon …", flush=True) # 计划任务不继承 PATH。daemon.exe 依赖 runtime\lib 里的 librknn3_api.dll 等, # 找不到会以 0xC0000135 (rc=3221225781) 直接退出、一个字都不输出。 rt = os.path.join(ROOT, "runtime", "rknn3_110") for d in (os.path.join(rt, "lib"), os.path.join(rt, "bin")): if os.path.isdir(d) and d not in os.environ.get("PATH", ""): os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "") print(f"[env] PATH head = {os.environ['PATH'].split(os.pathsep)[:2]}", flush=True) dmn = Daemon() print("daemon ready", flush=True) class H(BaseHTTPRequestHandler): def _json(self, code, obj): raw = json.dumps(obj, ensure_ascii=False).encode() self.send_response(code) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(raw))) self.end_headers() self.wfile.write(raw) def do_POST(self): if self.path != "/v1/systemone": self.send_error(404) return n = int(self.headers.get("Content-Length", 0) or 0) if n > MAX_BODY: self._json(413, {"error": f"body too large: {n} bytes (limit {MAX_BODY})"}) return raw_in = self.rfile.read(n) if n > 0 else b"" if not raw_in: self._json(400, {"error": "empty body"}) return try: req = json.loads(raw_in) except json.JSONDecodeError as e: # 4xx 而不是 5xx:这是调用方的问题,回 500 会让人以为服务挂了 self._json(400, {"error": f"invalid JSON at line {e.lineno} col {e.colno}: {e.msg}"}) return try: validate_request(req) except BadRequest as e: self._json(400, {"error": str(e)}) return try: # 整个决策独占 daemon:中间的 K+1 次 prefill 不能被别的请求插进来 with DECIDE_LOCK: body = decide(dmn, sc, req["state"], req["questions"]) self._json(200, body) except Exception as e: # traceback 只进日志。响应里给调用方也没用,还泄漏实现细节。 with open(os.path.join(HERE, "serve_err.log"), "a", encoding="utf-8") as f: f.write(traceback.format_exc() + "\n") self._json(500, {"error": f"internal error: {type(e).__name__}: {e}"}) def do_GET(self): # 探活:计划任务起来后用它确认"服务真的能答" if self.path == "/healthz": raw = json.dumps({"ok": True, "model": "Kev-0.8B", "pid": os.getpid()}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(raw))) self.end_headers() self.wfile.write(raw) elif self.path == "/v1/systemone": # 存在但不接受 GET:405 比 404 准确,调用方一眼看出是方法错了 self.send_response(405) self.send_header("Allow", "POST") self.send_header("Content-Type", "application/json; charset=utf-8") raw = json.dumps({"error": "use POST"}).encode() self.send_header("Content-Length", str(len(raw))) self.end_headers() self.wfile.write(raw) else: self.send_error(404) def log_message(self, *a): pass port = int(os.environ.get("KEV_PORT", "17890")) host = os.environ.get("KEV_HOST", "127.0.0.1") try: srv = ThreadingHTTPServer((host, port), H) except OSError as e: # 端口被占是最常见的启动失败原因(旧进程没退干净),必须说清楚 print(f"BIND_FAILED {host}:{port} -> {e}", flush=True) raise SystemExit(2) print(f"listening on {host}:{port}", flush=True) srv.serve_forever() if __name__ == "__main__": main()