File size: 3,025 Bytes
95d305b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
"""CPU-only, single-checkpoint deployment of Laya's native Jev HTTP API."""

import os


MODEL = "convaiinnovations/laya-multilingual"
MODEL_REVISION = "1720e3e3357cfe1e281542e223f8273b0890ca34"
WORKERS = 2


def jev_result(result, confidence):
    """Keep Jev's confidence semantics; incomplete input must never hard-reject."""
    usage = result.get("usage") or {}
    truncated = bool(usage.get("truncated") or usage.get("state_tokens_dropped"))
    for answer in result["answers"].values():
        if answer.get("type") in {"choice", "score"}:
            value = confidence(answer)
            answer["confidence"] = 0.0 if truncated or value is None else round(value, 4)
    result["model"] = MODEL
    return result


def create_app():
    if not os.environ.get("LAYA_API_KEY", "").strip():
        raise RuntimeError("LAYA_API_KEY is required")
    os.environ.update(
        LAYA_DEVICE="cpu",
        LAYA_MAX_CONCURRENT="8",
        LAYA_JEV_STRICT="1",
        LAYA_IDLE_UNLOAD_SECONDS="0",
        LAYA_TOURNAMENT_GROUP="0",
        LAYA_MAX_TOKEN_BUDGET="4096",
        CUDA_VISIBLE_DEVICES="",
    )
    import torch
    from laya.confidence import jev_confidence
    from laya.router import Router
    from laya.serve import create_app as native_app

    torch.set_num_threads(max(1, min(4, int(os.environ.get("LAYA_THREADS", "2")))))

    class MultilingualRouter(Router):
        def resolve(self, name):
            if name == "typesafe-ai/jev":
                name = "multilingual"
            return super().resolve(name)

        def route(self, state, questions=None, **kwargs):
            return super().route(state, questions, model="multilingual")

        def predict(self, state, questions, **kwargs):
            kwargs.setdefault("max_len", 4096)
            return jev_result(super().predict(state, questions, **kwargs), jev_confidence)

    router = MultilingualRouter(
        device="cpu", standalone_repos=True, max_loaded=1,
        revisions={"multilingual": MODEL_REVISION},
    )
    # Removing the other registry entries also makes an accidental direct load fail closed.
    router.models = {"multilingual": MODEL}
    router.preload(["multilingual"])
    app = native_app(router=router)
    app.state.laya_router = router
    return app


if __name__ == "__main__":
    if os.environ.get("SPACES_ZERO_GPU", "").lower() in {"1", "t", "true"}:
        import spaces

        @spaces.GPU(duration=1)
        def unused_gpu():
            return True

        # shortcut: spaces 0.51.3 only invokes this from Gradio.launch; recheck on SDK upgrades.
        spaces.zero.startup()
        print("ZeroGPU startup registered; dummy GPU function not invoked", flush=True)

    import uvicorn

    # Shared CPU hosts can block worker pings for more than 5s while loading the checkpoint.
    uvicorn.run("app:create_app", factory=True, host="0.0.0.0",
                port=int(os.environ.get("PORT", "7860")), workers=WORKERS, access_log=False,
                timeout_worker_healthcheck=30)