PEFT
Safetensors
Chinese
kev
queue-position
decision-model
lora
sft
permutation-consistency
chinese
Eval Results (legacy)
Instructions to use fzkun/kev-queue-v4-4b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use fzkun/kev-queue-v4-4b with PEFT:
from peft import PeftModel from transformers import AutoModel base_model = AutoModel.from_pretrained("Qwen/Qwen3.5-4B-Base") model = PeftModel.from_pretrained(base_model, "fzkun/kev-queue-v4-4b") - Notebooks
- Google Colab
- Kaggle
Download deployment/service.queue.py from fzkun/kev-queue-v4-4b: direct link, hf CLI and curl.
- Browser
- Download file 4 kB
-
https://huggingface.co/fzkun/kev-queue-v4-4b/resolve/main/deployment/service.queue.py
- Command line
-
hf download hf://fzkun/kev-queue-v4-4b/deployment/service.queue.py
-
curl -L -o service.queue.py https://huggingface.co/fzkun/kev-queue-v4-4b/resolve/main/deployment/service.queue.py
4 kB
| import asyncio | |
| from functools import partial | |
| import os | |
| from pathlib import Path | |
| import time | |
| import torch | |
| def main(): | |
| secret_path = os.environ.get("KEV_API_KEY_FILE") | |
| if not secret_path: | |
| raise ValueError("KEV_API_KEY_FILE is required") | |
| secret = Path(secret_path).read_text(encoding="utf-8").strip() | |
| if not secret: | |
| raise ValueError("API key must not be empty") | |
| os.environ["KEV_API_KEY"] = secret | |
| fraction = float(os.environ.get("KEV_GPU_MEMORY_FRACTION", "0.23")) | |
| if not 0 < fraction <= 1: | |
| raise ValueError("Invalid GPU memory fraction") | |
| torch.cuda.set_per_process_memory_fraction(fraction, device=0) | |
| # Match queue-v4's evaluated inference configuration. | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| torch.backends.cudnn.allow_tf32 = False | |
| torch.backends.cuda.enable_flash_sdp(False) | |
| torch.backends.cuda.enable_mem_efficient_sdp(False) | |
| import kev.model as model | |
| window = int(os.environ.get("KEV_DEPLOY_MAX_STATE", "8192")) | |
| if not 1 <= window <= 8192: | |
| raise ValueError("Deployment supports at most 8192 tokens") | |
| model.SERVE_MAX_STATE = window | |
| model.SERVE_MAX_BRANCH = window | |
| model.SERVE_MAX_PACKED = window * 2 | |
| model.ROW_PASS_TOKENS = window | |
| model.rows_per_pass = partial(model.rows_per_pass, budget=window) | |
| from kev import serve | |
| serve.MODEL_NAMES = ("kev-queue-v4", "kev-latest", "jev-latest") | |
| # Keep evaluated short rows on the exact attention path. Use KEV's | |
| # existing memory-efficient context for longer rows to support 8K. | |
| from kev.model import rows_of | |
| from kev.predictors import long_row_kernels | |
| original_run = serve.Server._run | |
| def queue_run(server, encs): | |
| longest = 0 | |
| for enc in encs: | |
| state, _, rows = rows_of(enc) | |
| longest = max(longest, len(state) + max(len(r["ids"]) for r in rows)) | |
| if longest > 2048: | |
| with long_row_kernels(): | |
| return original_run(server, encs) | |
| # Native serving uses a prefix continuation and a batched head. BF16 | |
| # reassociation changed one tested position. Use exactly the evaluated | |
| # forward/readout on short rows; retain native efficient serving at 8K. | |
| serve.sync(server.device) | |
| started = time.perf_counter() | |
| probabilities = [[torch.softmax(z, -1).cpu() for z in server.model.forward(enc)] for enc in encs] | |
| serve.sync(server.device) | |
| latency = round(1000 * (time.perf_counter() - started), 1) | |
| server.batches += 1 | |
| server.batched_requests += len(encs) | |
| return [([q.tolist() for q in ps], { | |
| "tokens": len(enc["ids"]), "state_tokens": enc["state_tokens"], | |
| "state_tokens_used": enc["seg"].count(0), "latency_ms": latency, | |
| "prefix_cache_hit": False, | |
| }) for enc, ps in zip(encs, probabilities)] | |
| serve.Server._run = queue_run | |
| from fastapi.responses import JSONResponse | |
| inference_lock = asyncio.Lock() | |
| async def limit_inference(request, call_next): | |
| if request.method != "POST" or not request.url.path.startswith("/v1/systemone"): | |
| return await call_next(request) | |
| if inference_lock.locked(): | |
| return JSONResponse({"detail": "Inference busy; retry later"}, status_code=503, | |
| headers={"Retry-After": "1"}) | |
| async with inference_lock: | |
| return await call_next(request) | |
| def health(): | |
| server = getattr(serve.app.state, "server", None) | |
| if server is None: | |
| return JSONResponse({"status": "starting"}, status_code=503) | |
| return {"status": "ok", "device": server.device, "max_state_tokens": window, | |
| "checkpoint": server.checkpoint.requested, | |
| "temperature": server.model.head.temperature, "model": "kev-queue-v4"} | |
| serve.main() | |
| if __name__ == "__main__": | |
| main() | |