File size: 3,303 Bytes
ad67f6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """Jev-compatible /v1/systemone server that serves OUR fine-tuned checkpoint (model_soft).
laya-serve serves the upstream family; this one points straight at
output/model_soft so agents get the fine-tuned, holdout-calibrated model.
Bearer auth via env JEV_API_KEY (mandatory), binds 127.0.0.1 by default.
Run:
$env:JEV_API_KEY="..." ; D:\venvs\voxcpm-cuda\Scripts\python.exe d:\QoderCN\jev_finetune\serve_soft.py
Smoke:
see smoke_serve.ps1 next to this file
"""
from __future__ import annotations
import json
import os
import sys
import time
os.environ.setdefault("USE_TF", "0")
sys.stdout.reconfigure(encoding="utf-8")
WORK = os.path.dirname(os.path.abspath(__file__))
MODEL_DIR = os.environ.get("JEV_MODEL_DIR", os.path.join(WORK, "output", "model_soft"))
HOST = os.environ.get("JEV_HOST", "127.0.0.1")
PORT = int(os.environ.get("JEV_PORT", "8190"))
API_KEY = os.environ.get("JEV_API_KEY", "")
DECISIONS_LOG = os.path.join(WORK, "decisions.jsonl")
def _log_decision(state, questions, answers, latency_ms):
"""Append one JSONL record per request - the fuel for the domain fine-tune loop.
Never allowed to break the endpoint."""
try:
with open(DECISIONS_LOG, "a", encoding="utf-8") as f:
f.write(json.dumps({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "latency_ms": latency_ms,
"state": state, "questions": questions, "answers": answers},
ensure_ascii=False) + "\n")
except Exception:
pass
from fastapi import FastAPI, Header, HTTPException # noqa: E402
app = FastAPI(title="laya-soft systemone")
_agent = None
@app.on_event("startup")
def _load():
global _agent
import laya
t0 = time.time()
_agent = laya.Agent(MODEL_DIR, device="cuda")
_agent.predict({"ping": "hello"}, {"ok": {"type": "noul", "instructions": "is this a ping"}})
print(f"[serve] model loaded in {time.time()-t0:.1f}s from {MODEL_DIR}")
def _auth(authorization: str):
if not API_KEY:
raise HTTPException(500, "server started without JEV_API_KEY")
if authorization != f"Bearer {API_KEY}":
raise HTTPException(401, "invalid api key")
@app.post("/v1/systemone")
def systemone(body: dict, authorization: str = Header(default="")):
_auth(authorization)
state = body.get("state")
questions = body.get("questions")
if state is None or not isinstance(questions, dict):
raise HTTPException(422, "body must have 'state' and 'questions' (object)")
t0 = time.perf_counter()
res = _agent.predict(state, questions)
dt = (time.perf_counter() - t0) * 1000
ans = res["answers"] if isinstance(res, dict) and "answers" in res else res
_log_decision(state, questions, ans, round(dt, 1))
n_tok = len(json.dumps(state, ensure_ascii=False)) // 4 + len(json.dumps(questions)) // 4
return {"model": os.path.basename(MODEL_DIR), "answers": ans,
"usage": {"input_tokens": n_tok, "output_tokens": 0},
"latency_ms": round(dt, 1)}
@app.get("/health")
def health():
return {"ok": True, "model": MODEL_DIR}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host=HOST, port=PORT, log_level="warning")
|