File size: 4,000 Bytes
51ad825
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import asyncio
from functools import partial
import os
from pathlib import Path
import time

import torch


def main():
    secret_path = os.environ.get("KEV_API_KEY_FILE")
    if not secret_path:
        raise ValueError("KEV_API_KEY_FILE is required")
    secret = Path(secret_path).read_text(encoding="utf-8").strip()
    if not secret:
        raise ValueError("API key must not be empty")
    os.environ["KEV_API_KEY"] = secret
    fraction = float(os.environ.get("KEV_GPU_MEMORY_FRACTION", "0.23"))
    if not 0 < fraction <= 1:
        raise ValueError("Invalid GPU memory fraction")
    torch.cuda.set_per_process_memory_fraction(fraction, device=0)
    # Match queue-v4's evaluated inference configuration.
    torch.backends.cuda.matmul.allow_tf32 = False
    torch.backends.cudnn.allow_tf32 = False
    torch.backends.cuda.enable_flash_sdp(False)
    torch.backends.cuda.enable_mem_efficient_sdp(False)
    import kev.model as model

    window = int(os.environ.get("KEV_DEPLOY_MAX_STATE", "8192"))
    if not 1 <= window <= 8192:
        raise ValueError("Deployment supports at most 8192 tokens")
    model.SERVE_MAX_STATE = window
    model.SERVE_MAX_BRANCH = window
    model.SERVE_MAX_PACKED = window * 2
    model.ROW_PASS_TOKENS = window
    model.rows_per_pass = partial(model.rows_per_pass, budget=window)
    from kev import serve
    serve.MODEL_NAMES = ("kev-queue-v4", "kev-latest", "jev-latest")
    # Keep evaluated short rows on the exact attention path. Use KEV's
    # existing memory-efficient context for longer rows to support 8K.
    from kev.model import rows_of
    from kev.predictors import long_row_kernels
    original_run = serve.Server._run

    @torch.no_grad()
    def queue_run(server, encs):
        longest = 0
        for enc in encs:
            state, _, rows = rows_of(enc)
            longest = max(longest, len(state) + max(len(r["ids"]) for r in rows))
        if longest > 2048:
            with long_row_kernels():
                return original_run(server, encs)
        # Native serving uses a prefix continuation and a batched head. BF16
        # reassociation changed one tested position. Use exactly the evaluated
        # forward/readout on short rows; retain native efficient serving at 8K.
        serve.sync(server.device)
        started = time.perf_counter()
        probabilities = [[torch.softmax(z, -1).cpu() for z in server.model.forward(enc)] for enc in encs]
        serve.sync(server.device)
        latency = round(1000 * (time.perf_counter() - started), 1)
        server.batches += 1
        server.batched_requests += len(encs)
        return [([q.tolist() for q in ps], {
            "tokens": len(enc["ids"]), "state_tokens": enc["state_tokens"],
            "state_tokens_used": enc["seg"].count(0), "latency_ms": latency,
            "prefix_cache_hit": False,
        }) for enc, ps in zip(encs, probabilities)]

    serve.Server._run = queue_run
    from fastapi.responses import JSONResponse

    inference_lock = asyncio.Lock()

    @serve.app.middleware("http")
    async def limit_inference(request, call_next):
        if request.method != "POST" or not request.url.path.startswith("/v1/systemone"):
            return await call_next(request)
        if inference_lock.locked():
            return JSONResponse({"detail": "Inference busy; retry later"}, status_code=503,
                                headers={"Retry-After": "1"})
        async with inference_lock:
            return await call_next(request)

    @serve.app.get("/health")
    def health():
        server = getattr(serve.app.state, "server", None)
        if server is None:
            return JSONResponse({"status": "starting"}, status_code=503)
        return {"status": "ok", "device": server.device, "max_state_tokens": window,
                "checkpoint": server.checkpoint.requested,
                "temperature": server.model.head.temperature, "model": "kev-queue-v4"}

    serve.main()


if __name__ == "__main__":
    main()