Kev-0.8B-RKNN3-RK1828 / host /kev_serve.py
jj90's picture
Add model card, deployment guide and host-side code
a846b3d verified
Raw History Blame Contribute Delete
18.6 kB
"""Kev 在 RK1828 上的在线决策服务(跑在 Windows)。
架构:
HTTP (本文件) --stdin/stdout--> daemon.exe --NPU--> logits
host 侧 numpy 做 反解 + 指针打分
一次请求 = K+1 次 prefill(<decide> 一次 + 每个 </opt> 一次),
因为图的输出只保留最后一个位置,取别的位置只能把序列截断重跑。
接口:POST /v1/systemone
{"state": <任意 JSON>, "questions": {"q1": {"type":"choice|noul|score",
"instructions": "...", "criteria": {...} | [...]}}}
"""
import base64
import json
import os
import subprocess
import sys
import threading
import time
import traceback
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import numpy as np
# 所有路径都从本文件位置推导,不写死盘符 —— 交付后整个目录搬到任何地方都能用。
# 布局:<包根>/kevrunner/kev_serve.py,<包根>/models/<模型>,<包根>/runtime/rknn3_110
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
# 两处可用环境变量覆盖,方便放进别的目录布局(不设时行为与原先完全一致)
MODEL = os.environ.get("KEV_MODEL_DIR") or os.path.join(ROOT, "models", "Kev-0.8B")
SEP = os.sep
WORK = HERE
DAEMON = os.environ.get("KEV_DAEMON") or os.path.join(
WORK, r"rknn3_session_test_demo\daemon.exe")
CUR = os.path.join(WORK, r"out\cur.bin")
# plan task has no console; pipe stdout EOFs immediately there, so use a file channel
CHAN = os.path.join(WORK, "chan.log")
# 官方 kev/model.py::SPECIAL 的 id
T_STATE, T_Q, T_OPT, T_OPT_END, T_DECIDE = 248060, 248061, 248049, 248050, 248062
VOCAB, HID = 248320, 1024
MAX_OPTS = 255
# ---------- 渲染(对齐官方 kev/api.py) ----------
def render(v, indent=0):
pad = " " * indent
if v is None:
return ""
if isinstance(v, (str, int, float, bool)):
return str(v)
if isinstance(v, list):
return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v)
return "\n".join(
f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list))
else f"{pad}{k}: {render(x)}" for k, x in v.items())
def option_text(name, desc):
return name if desc is None or desc == "" else f"{name}: {render(desc)}"
def question_options(q):
"""返回 (选项文本列表, 选项名列表, 题型)。"""
t = q["type"]
if t == "noul":
return ["No", "Yes"], ["false", "true"], "noul"
if t == "score":
crit = q["criteria"]
return [render(c) for c in crit], [str(i) for i in range(len(crit))], "score"
crit = q["criteria"]
names = list(crit)
return [option_text(k, crit[k]) for k in names], names, "choice"
def round_prob(x):
return round(float(x), 4)
def choice_confidence(p):
"""(p_max - 1/K) / (1 - 1/K):均匀分布为 0,确定时为 1。"""
K = len(p)
return 1.0 if K == 1 else (float(max(p)) - 1 / K) / (1 - 1 / K)
def score_confidence(p):
"""1 - E|level - mode| / D,D = 均匀分布的平均绝对偏差。"""
L = len(p)
if L == 1:
return 1.0
p = [float(x) for x in p]
mode = max(range(L), key=p.__getitem__)
D = sum(abs(i - (L - 1) / 2) for i in range(L)) / L
return max(0.0, 1.0 - sum(pi * abs(i - mode) for i, pi in enumerate(p)) / D)
# ---------- daemon 通信 ----------
class Daemon:
def __init__(self):
self.p = subprocess.Popen(
# 用 ntpath 拼:本文件在 Windows 上跑,os.path.join 会按 Windows 规则走,
# 但显式用 "\\" 更稳(曾被混合分隔符坑过)
[DAEMON]
+ [MODEL + SEP + f"Kev-0.8B{ext}"
for ext in (".rknn", ".weight", ".tokenizer.gguf", ".embed.bin")]
+ ["1024", "1", "0xff", "0", "0", "key.env"],
cwd=os.path.dirname(DAEMON),
stdin=subprocess.PIPE,
# 计划任务没有控制台,daemon 接管道时 stdout 会立刻 EOF
# (rc=None 但进程还活着)。落文件,有无控制台都成立。
stdout=open(CHAN, "w+b", buffering=0),
stderr=subprocess.STDOUT)
self.lock = threading.Lock()
self._buf = b""
self.offset = 0
self._buf = b""
# 等 daemon 在通道文件里写出 READY_DAEMON(模型加载约 30 s,给 180 s)
if not self._wait_for("READY_DAEMON", 180):
raise RuntimeError(
f"daemon 180s 内没有 READY_DAEMON(rc={self.p.poll()}),"
f"最后输出:{self._tail() or '(空)'}")
def _tail(self, n=6):
try:
with open(CHAN, "r", encoding="utf-8", errors="replace") as f:
return " | ".join(l.rstrip()[:110] for l in f.readlines()[-n:])
except OSError:
return ""
def _wait_for(self, prefix, timeout):
"""轮询通道文件,从 self.offset 往后找以 prefix 开头的行。
用文件而不是管道:计划任务没有控制台,管道方案在那里会立刻 EOF。
"""
t0 = time.time()
while time.time() - t0 < timeout:
chunk = b""
try:
with open(CHAN, "rb") as f:
f.seek(self.offset)
chunk = f.read()
except OSError:
pass
if chunk:
self.offset += len(chunk)
buf = self._buf + chunk
parts = buf.split(b"\n")
self._buf = parts.pop() # 不完整的尾巴留到下次
for raw in parts:
ln = raw.decode("utf-8", "replace").rstrip()
if ln.startswith(prefix):
return ln
if self.p.poll() is not None:
return None
time.sleep(0.05)
return None
def _cmd(self, line, stop_prefix):
self.p.stdin.write((line + "\n").encode("utf-8"))
self.p.stdin.flush()
out = self._wait_for(stop_prefix, 180)
if out is None:
raise RuntimeError(
f"等 {stop_prefix} 超时/退出(rc={self.p.poll()}):{self._tail()}")
return out
def tok(self, text):
b64 = base64.b64encode(text.encode("utf-8")).decode("ascii")
with self.lock:
out = self._cmd("TOK " + b64, "IDS ")
parts = out.split()
n = int(parts[1])
return [int(x) for x in parts[2:2 + n]]
def prefill(self, ids, append=False):
"""跑一次 prefill,返回 248320 维 logits。
append=False → R(先清 kvcache,从头算)
append=True → A(接在上一次之后,只算新增这段)
实测两者余弦 ≥ 0.998,top-1 一致,所以同一题内可以安全用增量。
"""
with self.lock:
self._cmd(("A " if append else "R ") + " ".join(str(x) for x in ids), "DONE")
return np.fromfile(CUR, dtype=np.float16).astype(np.float32)
def close(self):
try:
self.p.stdin.write(b"quit\n")
self.p.stdin.flush()
except Exception:
pass
try:
self.p.wait(timeout=10)
except Exception:
self.p.kill()
# ---------- 打分层 ----------
class Scorer:
def __init__(self, assets):
self.Ginv = np.load(os.path.join(assets, "Ginv.npy")) # 1024x1024
self.q = np.load(os.path.join(assets, "head_q.npy")) # 256x1024
self.k = np.load(os.path.join(assets, "head_k.npy")) # 256x1024
self.T = float(np.load(os.path.join(assets, "temperature.npy"))[0])
# 一次性读进内存(500 MB fp16)。用 memmap 的话每次 hidden() 都要从磁盘
# 重新读一遍 500 MB —— 实测把单请求从 7.6 s 拖到很慢。
print("loading embed weights to RAM …", flush=True)
W16 = np.fromfile(os.path.join(MODEL, "Kev-0.8B.embed.bin"),
dtype=np.float16).reshape(VOCAB, HID)
# 预转 float32 常驻(1 GB)。否则每次 hidden() 都要转 2.68 亿个元素,
# 实测单请求 7.6 s 里大部分耗在这上面。
print("casting weights to fp32 …", flush=True)
self.W = np.ascontiguousarray(W16, dtype=np.float32)
del W16
def hidden(self, logits):
"""h = (WᵀW)⁻¹ Wᵀ logits"""
# Wᵀ @ logits:memmap 分块算,避免一次性 500 MB 上浮
rhs = np.zeros(HID, dtype=np.float32)
step = 16384
for s in range(0, VOCAB, step):
e = min(s + step, VOCAB)
rhs += self.W[s:e].T @ logits[s:e]
return self.Ginv @ rhs
def scores(self, h_decide, h_opts):
z = (h_opts @ self.k.T) @ (self.q @ h_decide) / np.sqrt(self.q.shape[0])
z = z / self.T
z = z - z.max()
p = np.exp(z)
return p / p.sum()
# ---------- 决策 ----------
class BadRequest(Exception):
"""客户端参数错误 → 400。和「服务出故障」必须分开:
参数错回 500 会让人以为服务挂了,日志里也全是假异常。"""
def validate_request(req):
"""把能判的错都在**进模型之前**判掉。
这些情况以前会一路走到 question_options 里抛 KeyError,
被外层 catch 成 500 并回传 traceback —— 既误导调用方,又泄漏实现细节。
还有两处更糟的:缺 state 会白跑一次 prefill(几百毫秒),
未知 type 会被静默当成 choice 处理(拼错 "choices" 也能拿到答案)。
"""
if not isinstance(req, dict):
raise BadRequest("body must be a JSON object")
state = req.get("state")
# state 允许三种形态,这是 decision-v2 的真实分布,不是想当然:
# str 664 条 / dict 440 条 / list 72 条
# 一开始只允许 str,结果把 512 条合法请求挡成 400 —— 校验过严比不校验更糟。
if state is None:
raise BadRequest("state is required")
if isinstance(state, str):
if not state.strip():
raise BadRequest("state must not be empty")
elif isinstance(state, (dict, list)):
if not state:
raise BadRequest("state must not be empty")
else:
raise BadRequest(
f"state must be a string, object or array, got {type(state).__name__}")
qs = req.get("questions")
if not isinstance(qs, dict):
raise BadRequest("questions is required: an object {id: question}")
if not qs:
# 空 questions 以前返回 {"answers": {}} —— 拼错字段名(比如写成 question)
# 会静默"成功",调用方以为模型没意见。宁可报错。
raise BadRequest("questions must not be empty")
for qid, q in qs.items():
if not isinstance(q, dict):
raise BadRequest(f"questions[{qid}] must be an object")
kind = q.get("type")
if kind not in ("noul", "choice", "score"):
raise BadRequest(
f"questions[{qid}].type must be one of noul/choice/score, got {kind!r}")
if kind == "choice":
crit = q.get("criteria")
if not isinstance(crit, dict) or not crit:
raise BadRequest(f"questions[{qid}].criteria must be a non-empty object")
elif kind == "score":
crit = q.get("criteria")
if not isinstance(crit, list) or not crit:
raise BadRequest(f"questions[{qid}].criteria must be a non-empty array")
# 一次 decide 会连续调用多次 prefill(K+1 次),中间必须独占 daemon。
# 只在单次 _cmd 里加锁是不够的 —— HTTP 层是多线程的,两个请求会交替
# 使用同一份 kvcache,后一个的 append 会接在前一个的序列后面,结果全乱。
DECIDE_LOCK = threading.Lock()
MAX_BODY = 1 << 20 # 1 MB,够放很长的 state,也挡住恶意大包
def decide(dmn, sc, state, questions):
st_ids = dmn.tok(render(state))
S = [T_STATE] + st_ids
out = {}
for qid, q in questions.items():
texts, names, kind = question_options(q)
K = len(texts)
if not 1 <= K <= MAX_OPTS:
out[qid] = {"error": f"options must be 1..{MAX_OPTS}"}
continue
ins = [T_Q] + dmn.tok(render(q.get("instructions") or ""))
spans = [[T_OPT] + dmn.tok(t) + [T_OPT_END] for t in texts]
ids = S + ins + [x for sp in spans for x in sp] + [T_DECIDE]
dec = len(ids) - 1
ends, cur = [], len(S) + len(ins)
for sp in spans:
ends.append(cur + len(sp) - 1)
cur += len(sp)
# 增量 prefill:只算一次全序列,其余位置靠逐段追加。
# 全量要算 sum(len)=约 K+1 倍的 token;增量只算整条序列一遍。
prev, h_opts = -1, []
for i, e in enumerate(ends):
h_opts.append(sc.hidden(dmn.prefill(ids[prev + 1:e + 1], append=(i > 0))))
prev = e
h_dec = sc.hidden(dmn.prefill(ids[prev + 1:dec + 1], append=True))
h_opts = np.stack(h_opts)
prob = sc.scores(h_dec, h_opts)
# 输出结构对齐官方 kev/api.py::to_answers
if kind == "noul":
out[qid] = {"type": "noul", "noul": round_prob(prob[1]),
"answer": bool(prob[1] >= 0.5)}
elif kind == "score":
lv = sum(i * pi for i, pi in enumerate(prob))
out[qid] = {"type": "score", "score": round_prob(lv),
"legend": texts,
"probabilities": {str(i): round_prob(v) for i, v in enumerate(prob)},
"confidence": round_prob(score_confidence(prob)),
"argmax": int(prob.argmax())}
else:
i = int(prob.argmax())
out[qid] = {"type": "choice", "answer": names[i],
"confidence": round_prob(choice_confidence(prob)),
"probabilities": {n: round_prob(p) for n, p in zip(names, prob)}}
return {"answers": out}
def main():
assets = os.path.join(HERE, "serve_assets")
print("loading assets …", flush=True)
sc = Scorer(assets)
print("starting daemon …", flush=True)
# 计划任务不继承 PATH。daemon.exe 依赖 runtime\lib 里的 librknn3_api.dll 等,
# 找不到会以 0xC0000135 (rc=3221225781) 直接退出、一个字都不输出。
rt = os.path.join(ROOT, "runtime", "rknn3_110")
for d in (os.path.join(rt, "lib"), os.path.join(rt, "bin")):
if os.path.isdir(d) and d not in os.environ.get("PATH", ""):
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
print(f"[env] PATH head = {os.environ['PATH'].split(os.pathsep)[:2]}", flush=True)
dmn = Daemon()
print("daemon ready", flush=True)
class H(BaseHTTPRequestHandler):
def _json(self, code, obj):
raw = json.dumps(obj, ensure_ascii=False).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def do_POST(self):
if self.path != "/v1/systemone":
self.send_error(404)
return
n = int(self.headers.get("Content-Length", 0) or 0)
if n > MAX_BODY:
self._json(413, {"error": f"body too large: {n} bytes (limit {MAX_BODY})"})
return
raw_in = self.rfile.read(n) if n > 0 else b""
if not raw_in:
self._json(400, {"error": "empty body"})
return
try:
req = json.loads(raw_in)
except json.JSONDecodeError as e:
# 4xx 而不是 5xx:这是调用方的问题,回 500 会让人以为服务挂了
self._json(400, {"error": f"invalid JSON at line {e.lineno} col {e.colno}: {e.msg}"})
return
try:
validate_request(req)
except BadRequest as e:
self._json(400, {"error": str(e)})
return
try:
# 整个决策独占 daemon:中间的 K+1 次 prefill 不能被别的请求插进来
with DECIDE_LOCK:
body = decide(dmn, sc, req["state"], req["questions"])
self._json(200, body)
except Exception as e:
# traceback 只进日志。响应里给调用方也没用,还泄漏实现细节。
with open(os.path.join(HERE, "serve_err.log"), "a", encoding="utf-8") as f:
f.write(traceback.format_exc() + "\n")
self._json(500, {"error": f"internal error: {type(e).__name__}: {e}"})
def do_GET(self):
# 探活:计划任务起来后用它确认"服务真的能答"
if self.path == "/healthz":
raw = json.dumps({"ok": True, "model": "Kev-0.8B",
"pid": os.getpid()}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
elif self.path == "/v1/systemone":
# 存在但不接受 GET:405 比 404 准确,调用方一眼看出是方法错了
self.send_response(405)
self.send_header("Allow", "POST")
self.send_header("Content-Type", "application/json; charset=utf-8")
raw = json.dumps({"error": "use POST"}).encode()
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
else:
self.send_error(404)
def log_message(self, *a):
pass
port = int(os.environ.get("KEV_PORT", "17890"))
host = os.environ.get("KEV_HOST", "127.0.0.1")
try:
srv = ThreadingHTTPServer((host, port), H)
except OSError as e:
# 端口被占是最常见的启动失败原因(旧进程没退干净),必须说清楚
print(f"BIND_FAILED {host}:{port} -> {e}", flush=True)
raise SystemExit(2)
print(f"listening on {host}:{port}", flush=True)
srv.serve_forever()
if __name__ == "__main__":
main()