import asyncio from functools import partial import os from pathlib import Path import time import torch def main(): secret_path = os.environ.get("KEV_API_KEY_FILE") if not secret_path: raise ValueError("KEV_API_KEY_FILE is required") secret = Path(secret_path).read_text(encoding="utf-8").strip() if not secret: raise ValueError("API key must not be empty") os.environ["KEV_API_KEY"] = secret fraction = float(os.environ.get("KEV_GPU_MEMORY_FRACTION", "0.23")) if not 0 < fraction <= 1: raise ValueError("Invalid GPU memory fraction") torch.cuda.set_per_process_memory_fraction(fraction, device=0) # Match queue-v4's evaluated inference configuration. torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False torch.backends.cuda.enable_flash_sdp(False) torch.backends.cuda.enable_mem_efficient_sdp(False) import kev.model as model window = int(os.environ.get("KEV_DEPLOY_MAX_STATE", "8192")) if not 1 <= window <= 8192: raise ValueError("Deployment supports at most 8192 tokens") model.SERVE_MAX_STATE = window model.SERVE_MAX_BRANCH = window model.SERVE_MAX_PACKED = window * 2 model.ROW_PASS_TOKENS = window model.rows_per_pass = partial(model.rows_per_pass, budget=window) from kev import serve serve.MODEL_NAMES = ("kev-queue-v4", "kev-latest", "jev-latest") # Keep evaluated short rows on the exact attention path. Use KEV's # existing memory-efficient context for longer rows to support 8K. from kev.model import rows_of from kev.predictors import long_row_kernels original_run = serve.Server._run @torch.no_grad() def queue_run(server, encs): longest = 0 for enc in encs: state, _, rows = rows_of(enc) longest = max(longest, len(state) + max(len(r["ids"]) for r in rows)) if longest > 2048: with long_row_kernels(): return original_run(server, encs) # Native serving uses a prefix continuation and a batched head. BF16 # reassociation changed one tested position. Use exactly the evaluated # forward/readout on short rows; retain native efficient serving at 8K. serve.sync(server.device) started = time.perf_counter() probabilities = [[torch.softmax(z, -1).cpu() for z in server.model.forward(enc)] for enc in encs] serve.sync(server.device) latency = round(1000 * (time.perf_counter() - started), 1) server.batches += 1 server.batched_requests += len(encs) return [([q.tolist() for q in ps], { "tokens": len(enc["ids"]), "state_tokens": enc["state_tokens"], "state_tokens_used": enc["seg"].count(0), "latency_ms": latency, "prefix_cache_hit": False, }) for enc, ps in zip(encs, probabilities)] serve.Server._run = queue_run from fastapi.responses import JSONResponse inference_lock = asyncio.Lock() @serve.app.middleware("http") async def limit_inference(request, call_next): if request.method != "POST" or not request.url.path.startswith("/v1/systemone"): return await call_next(request) if inference_lock.locked(): return JSONResponse({"detail": "Inference busy; retry later"}, status_code=503, headers={"Retry-After": "1"}) async with inference_lock: return await call_next(request) @serve.app.get("/health") def health(): server = getattr(serve.app.state, "server", None) if server is None: return JSONResponse({"status": "starting"}, status_code=503) return {"status": "ok", "device": server.device, "max_state_tokens": window, "checkpoint": server.checkpoint.requested, "temperature": server.model.head.temperature, "model": "kev-queue-v4"} serve.main() if __name__ == "__main__": main()