scripts/serve_soft.py

"""Jev-compatible /v1/systemone server that serves OUR fine-tuned checkpoint (model_soft).

laya-serve serves the upstream family; this one points straight at
output/model_soft so agents get the fine-tuned, holdout-calibrated model.
Bearer auth via env JEV_API_KEY (mandatory), binds 127.0.0.1 by default.

Run:
  $env:JEV_API_KEY="..." ; D:\venvs\voxcpm-cuda\Scripts\python.exe d:\QoderCN\jev_finetune\serve_soft.py
Smoke:
  see smoke_serve.ps1 next to this file
"""
from __future__ import annotations

import json
import os
import sys
import time

os.environ.setdefault("USE_TF", "0")
sys.stdout.reconfigure(encoding="utf-8")

WORK = os.path.dirname(os.path.abspath(__file__))
MODEL_DIR = os.environ.get("JEV_MODEL_DIR", os.path.join(WORK, "output", "model_soft"))
HOST = os.environ.get("JEV_HOST", "127.0.0.1")
PORT = int(os.environ.get("JEV_PORT", "8190"))
API_KEY = os.environ.get("JEV_API_KEY", "")
DECISIONS_LOG = os.path.join(WORK, "decisions.jsonl")


def _log_decision(state, questions, answers, latency_ms):
    """Append one JSONL record per request - the fuel for the domain fine-tune loop.
    Never allowed to break the endpoint."""
    try:
        with open(DECISIONS_LOG, "a", encoding="utf-8") as f:
            f.write(json.dumps({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "latency_ms": latency_ms,
                                "state": state, "questions": questions, "answers": answers},
                               ensure_ascii=False) + "\n")
    except Exception:
        pass

from fastapi import FastAPI, Header, HTTPException  # noqa: E402

app = FastAPI(title="laya-soft systemone")
_agent = None


@app.on_event("startup")
def _load():
    global _agent
    import laya

    t0 = time.time()
    _agent = laya.Agent(MODEL_DIR, device="cuda")
    _agent.predict({"ping": "hello"}, {"ok": {"type": "noul", "instructions": "is this a ping"}})
    print(f"[serve] model loaded in {time.time()-t0:.1f}s from {MODEL_DIR}")


def _auth(authorization: str):
    if not API_KEY:
        raise HTTPException(500, "server started without JEV_API_KEY")
    if authorization != f"Bearer {API_KEY}":
        raise HTTPException(401, "invalid api key")


@app.post("/v1/systemone")
def systemone(body: dict, authorization: str = Header(default="")):
    _auth(authorization)
    state = body.get("state")
    questions = body.get("questions")
    if state is None or not isinstance(questions, dict):
        raise HTTPException(422, "body must have 'state' and 'questions' (object)")
    t0 = time.perf_counter()
    res = _agent.predict(state, questions)
    dt = (time.perf_counter() - t0) * 1000
    ans = res["answers"] if isinstance(res, dict) and "answers" in res else res
    _log_decision(state, questions, ans, round(dt, 1))
    n_tok = len(json.dumps(state, ensure_ascii=False)) // 4 + len(json.dumps(questions)) // 4
    return {"model": os.path.basename(MODEL_DIR), "answers": ans,
            "usage": {"input_tokens": n_tok, "output_tokens": 0},
            "latency_ms": round(dt, 1)}


@app.get("/health")
def health():
    return {"ok": True, "model": MODEL_DIR}


if __name__ == "__main__":
    import uvicorn

    uvicorn.run(app, host=HOST, port=PORT, log_level="warning")