PEFT
Safetensors
Chinese
kev
queue-position
decision-model
lora
sft
permutation-consistency
chinese
Eval Results (legacy)
Instructions to use fzkun/kev-queue-v4-4b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use fzkun/kev-queue-v4-4b with PEFT:
from peft import PeftModel from transformers import AutoModel base_model = AutoModel.from_pretrained("Qwen/Qwen3.5-4B-Base") model = PeftModel.from_pretrained(base_model, "fzkun/kev-queue-v4-4b") - Notebooks
- Google Colab
- Kaggle
File size: 4,000 Bytes
51ad825 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | import asyncio
from functools import partial
import os
from pathlib import Path
import time
import torch
def main():
secret_path = os.environ.get("KEV_API_KEY_FILE")
if not secret_path:
raise ValueError("KEV_API_KEY_FILE is required")
secret = Path(secret_path).read_text(encoding="utf-8").strip()
if not secret:
raise ValueError("API key must not be empty")
os.environ["KEV_API_KEY"] = secret
fraction = float(os.environ.get("KEV_GPU_MEMORY_FRACTION", "0.23"))
if not 0 < fraction <= 1:
raise ValueError("Invalid GPU memory fraction")
torch.cuda.set_per_process_memory_fraction(fraction, device=0)
# Match queue-v4's evaluated inference configuration.
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.backends.cuda.enable_flash_sdp(False)
torch.backends.cuda.enable_mem_efficient_sdp(False)
import kev.model as model
window = int(os.environ.get("KEV_DEPLOY_MAX_STATE", "8192"))
if not 1 <= window <= 8192:
raise ValueError("Deployment supports at most 8192 tokens")
model.SERVE_MAX_STATE = window
model.SERVE_MAX_BRANCH = window
model.SERVE_MAX_PACKED = window * 2
model.ROW_PASS_TOKENS = window
model.rows_per_pass = partial(model.rows_per_pass, budget=window)
from kev import serve
serve.MODEL_NAMES = ("kev-queue-v4", "kev-latest", "jev-latest")
# Keep evaluated short rows on the exact attention path. Use KEV's
# existing memory-efficient context for longer rows to support 8K.
from kev.model import rows_of
from kev.predictors import long_row_kernels
original_run = serve.Server._run
@torch.no_grad()
def queue_run(server, encs):
longest = 0
for enc in encs:
state, _, rows = rows_of(enc)
longest = max(longest, len(state) + max(len(r["ids"]) for r in rows))
if longest > 2048:
with long_row_kernels():
return original_run(server, encs)
# Native serving uses a prefix continuation and a batched head. BF16
# reassociation changed one tested position. Use exactly the evaluated
# forward/readout on short rows; retain native efficient serving at 8K.
serve.sync(server.device)
started = time.perf_counter()
probabilities = [[torch.softmax(z, -1).cpu() for z in server.model.forward(enc)] for enc in encs]
serve.sync(server.device)
latency = round(1000 * (time.perf_counter() - started), 1)
server.batches += 1
server.batched_requests += len(encs)
return [([q.tolist() for q in ps], {
"tokens": len(enc["ids"]), "state_tokens": enc["state_tokens"],
"state_tokens_used": enc["seg"].count(0), "latency_ms": latency,
"prefix_cache_hit": False,
}) for enc, ps in zip(encs, probabilities)]
serve.Server._run = queue_run
from fastapi.responses import JSONResponse
inference_lock = asyncio.Lock()
@serve.app.middleware("http")
async def limit_inference(request, call_next):
if request.method != "POST" or not request.url.path.startswith("/v1/systemone"):
return await call_next(request)
if inference_lock.locked():
return JSONResponse({"detail": "Inference busy; retry later"}, status_code=503,
headers={"Retry-After": "1"})
async with inference_lock:
return await call_next(request)
@serve.app.get("/health")
def health():
server = getattr(serve.app.state, "server", None)
if server is None:
return JSONResponse({"status": "starting"}, status_code=503)
return {"status": "ok", "device": server.device, "max_state_tokens": window,
"checkpoint": server.checkpoint.requested,
"temperature": server.model.head.temperature, "model": "kev-queue-v4"}
serve.main()
if __name__ == "__main__":
main()
|