bolikoto's picture
feat(laya): deploy app.py (WORKERS=2, max_len=4096)
95d305b verified
Raw History Blame Contribute Delete
3.03 kB
"""CPU-only, single-checkpoint deployment of Laya's native Jev HTTP API."""
import os
MODEL = "convaiinnovations/laya-multilingual"
MODEL_REVISION = "1720e3e3357cfe1e281542e223f8273b0890ca34"
WORKERS = 2
def jev_result(result, confidence):
"""Keep Jev's confidence semantics; incomplete input must never hard-reject."""
usage = result.get("usage") or {}
truncated = bool(usage.get("truncated") or usage.get("state_tokens_dropped"))
for answer in result["answers"].values():
if answer.get("type") in {"choice", "score"}:
value = confidence(answer)
answer["confidence"] = 0.0 if truncated or value is None else round(value, 4)
result["model"] = MODEL
return result
def create_app():
if not os.environ.get("LAYA_API_KEY", "").strip():
raise RuntimeError("LAYA_API_KEY is required")
os.environ.update(
LAYA_DEVICE="cpu",
LAYA_MAX_CONCURRENT="8",
LAYA_JEV_STRICT="1",
LAYA_IDLE_UNLOAD_SECONDS="0",
LAYA_TOURNAMENT_GROUP="0",
LAYA_MAX_TOKEN_BUDGET="4096",
CUDA_VISIBLE_DEVICES="",
)
import torch
from laya.confidence import jev_confidence
from laya.router import Router
from laya.serve import create_app as native_app
torch.set_num_threads(max(1, min(4, int(os.environ.get("LAYA_THREADS", "2")))))
class MultilingualRouter(Router):
def resolve(self, name):
if name == "typesafe-ai/jev":
name = "multilingual"
return super().resolve(name)
def route(self, state, questions=None, **kwargs):
return super().route(state, questions, model="multilingual")
def predict(self, state, questions, **kwargs):
kwargs.setdefault("max_len", 4096)
return jev_result(super().predict(state, questions, **kwargs), jev_confidence)
router = MultilingualRouter(
device="cpu", standalone_repos=True, max_loaded=1,
revisions={"multilingual": MODEL_REVISION},
)
# Removing the other registry entries also makes an accidental direct load fail closed.
router.models = {"multilingual": MODEL}
router.preload(["multilingual"])
app = native_app(router=router)
app.state.laya_router = router
return app
if __name__ == "__main__":
if os.environ.get("SPACES_ZERO_GPU", "").lower() in {"1", "t", "true"}:
import spaces
@spaces.GPU(duration=1)
def unused_gpu():
return True
# shortcut: spaces 0.51.3 only invokes this from Gradio.launch; recheck on SDK upgrades.
spaces.zero.startup()
print("ZeroGPU startup registered; dummy GPU function not invoked", flush=True)
import uvicorn
# Shared CPU hosts can block worker pings for more than 5s while loading the checkpoint.
uvicorn.run("app:create_app", factory=True, host="0.0.0.0",
port=int(os.environ.get("PORT", "7860")), workers=WORKERS, access_log=False,
timeout_worker_healthcheck=30)