"""CPU-only, single-checkpoint deployment of Laya's native Jev HTTP API.""" import os MODEL = "convaiinnovations/laya-multilingual" MODEL_REVISION = "1720e3e3357cfe1e281542e223f8273b0890ca34" WORKERS = 2 def jev_result(result, confidence): """Keep Jev's confidence semantics; incomplete input must never hard-reject.""" usage = result.get("usage") or {} truncated = bool(usage.get("truncated") or usage.get("state_tokens_dropped")) for answer in result["answers"].values(): if answer.get("type") in {"choice", "score"}: value = confidence(answer) answer["confidence"] = 0.0 if truncated or value is None else round(value, 4) result["model"] = MODEL return result def create_app(): if not os.environ.get("LAYA_API_KEY", "").strip(): raise RuntimeError("LAYA_API_KEY is required") os.environ.update( LAYA_DEVICE="cpu", LAYA_MAX_CONCURRENT="8", LAYA_JEV_STRICT="1", LAYA_IDLE_UNLOAD_SECONDS="0", LAYA_TOURNAMENT_GROUP="0", LAYA_MAX_TOKEN_BUDGET="4096", CUDA_VISIBLE_DEVICES="", ) import torch from laya.confidence import jev_confidence from laya.router import Router from laya.serve import create_app as native_app torch.set_num_threads(max(1, min(4, int(os.environ.get("LAYA_THREADS", "2"))))) class MultilingualRouter(Router): def resolve(self, name): if name == "typesafe-ai/jev": name = "multilingual" return super().resolve(name) def route(self, state, questions=None, **kwargs): return super().route(state, questions, model="multilingual") def predict(self, state, questions, **kwargs): kwargs.setdefault("max_len", 4096) return jev_result(super().predict(state, questions, **kwargs), jev_confidence) router = MultilingualRouter( device="cpu", standalone_repos=True, max_loaded=1, revisions={"multilingual": MODEL_REVISION}, ) # Removing the other registry entries also makes an accidental direct load fail closed. router.models = {"multilingual": MODEL} router.preload(["multilingual"]) app = native_app(router=router) app.state.laya_router = router return app if __name__ == "__main__": if os.environ.get("SPACES_ZERO_GPU", "").lower() in {"1", "t", "true"}: import spaces @spaces.GPU(duration=1) def unused_gpu(): return True # shortcut: spaces 0.51.3 only invokes this from Gradio.launch; recheck on SDK upgrades. spaces.zero.startup() print("ZeroGPU startup registered; dummy GPU function not invoked", flush=True) import uvicorn # Shared CPU hosts can block worker pings for more than 5s while loading the checkpoint. uvicorn.run("app:create_app", factory=True, host="0.0.0.0", port=int(os.environ.get("PORT", "7860")), workers=WORKERS, access_log=False, timeout_worker_healthcheck=30)