fix(laya): 迁移普通CPU Space并保持四worker并发32
Browse files- README.md +58 -8
- app.py +81 -0
- requirements.txt +4 -0
README.md
CHANGED
|
@@ -1,13 +1,63 @@
|
|
| 1 |
---
|
| 2 |
-
title: Laya Multilingual
|
| 3 |
-
emoji: 🏆
|
| 4 |
-
colorFrom: red
|
| 5 |
-
colorTo: purple
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Laya Multilingual CPU API
|
|
|
|
|
|
|
|
|
|
| 3 |
sdk: gradio
|
| 4 |
+
sdk_version: 6.24.0
|
|
|
|
| 5 |
app_file: app.py
|
| 6 |
+
python_version: "3.10"
|
| 7 |
+
license: apache-2.0
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# Laya multilingual CPU API
|
| 11 |
+
|
| 12 |
+
Only `convaiinnovations/laya-multilingual` is loaded, pinned to the revision in
|
| 13 |
+
`app.py`. CPU is forced even on ZeroGPU. The unused GPU declaration is never
|
| 14 |
+
called and is not exposed as an API. HF runs four server workers, each admitting
|
| 15 |
+
at most eight inference requests (including its bounded inference queue), for a
|
| 16 |
+
total ceiling of 32. InstaBox keeps one worker admitting at most 16 requests.
|
| 17 |
+
Excess requests receive HTTP 503 and `Retry-After: 1`; a full worker can reject
|
| 18 |
+
even when another worker has a free slot. Actual CPU inference is serialized
|
| 19 |
+
within each worker by Laya's native server to avoid oversubscription and memory spikes.
|
| 20 |
+
|
| 21 |
+
Only on ZeroGPU, ASGI startup with Spaces SDK 0.51.3 explicitly runs the SDK's startup-report
|
| 22 |
+
callback (normally called by Gradio.launch), not its GPU scheduling API. The
|
| 23 |
+
dummy remains uncalled; recheck this callback when the platform upgrades Spaces.
|
| 24 |
+
|
| 25 |
+
`POST /v1/systemone` accepts the existing Jev `model`, `state`, `questions` JSON
|
| 26 |
+
and returns only `model`, `answers`, `usage`. `LAYA_API_KEY` is mandatory and must
|
| 27 |
+
be sent as a Bearer token. `/health` is public liveness only; an authenticated
|
| 28 |
+
request includes the loaded checkpoint, revision and actual CPU device.
|
| 29 |
+
Jev confidence is calculated with Laya's compatibility helper; truncated input
|
| 30 |
+
has confidence zero, preserving Opportunity's existing fail-open gate.
|
| 31 |
+
|
| 32 |
+
## InstaBox (persistent installation)
|
| 33 |
+
|
| 34 |
+
Install `app.py`, `requirements.txt`, `.venv`, and the model cache below
|
| 35 |
+
`/data/workspace/laya`. Keep `LAYA_API_KEY` in the mode-0600 `.env` there, never
|
| 36 |
+
in Git or command-line arguments. Link `laya.service` below the persistent
|
| 37 |
+
`~/.config/systemd/user/` and enable it with `systemctl --user`. Its default-target
|
| 38 |
+
link is persistent too; InstaBox enables user linger during boot. Publish
|
| 39 |
+
port 7860 with InstaBox's HTTPS port forwarding; never expose an unauthenticated
|
| 40 |
+
inference listener. Do not restart unrelated services to test boot persistence.
|
| 41 |
+
|
| 42 |
+
On InstaBox, pre-install `torch==2.8.0+cpu` from PyTorch's CPU wheel index before
|
| 43 |
+
installing `requirements.txt`; it satisfies `torch==2.8.0` without CUDA packages.
|
| 44 |
+
The Space uses HF's supported Torch declaration but never places a tensor on GPU.
|
| 45 |
+
|
| 46 |
+
## Hugging Face
|
| 47 |
+
|
| 48 |
+
Create a new Gradio Space on `cpu-basic`; upload only these three Space files
|
| 49 |
+
(`app.py`, `requirements.txt`, `README.md`) and set its `LAYA_API_KEY` secret.
|
| 50 |
+
Do not upload `.env`, caches, units, or virtual environments. This runs on CPU,
|
| 51 |
+
without registering ZeroGPU startup or invoking the dummy GPU function. Preserve
|
| 52 |
+
the existing Space variables (`LAYA_THREADS=4`, `TOKENIZERS_PARALLELISM=false`).
|
| 53 |
+
Never silently substitute paid hardware. CPU Basic sleeps after 48 hours of inactivity.
|
| 54 |
+
|
| 55 |
+
## Opportunity
|
| 56 |
+
|
| 57 |
+
Set all four runtime secret-group variables together:
|
| 58 |
+
`LAYA_INSTABOX_ENDPOINT`, `LAYA_INSTABOX_API_KEY`, `LAYA_HF_ENDPOINT`,
|
| 59 |
+
`LAYA_HF_API_KEY`. The endpoints are HTTPS `/v1/systemone` URLs. This opts the
|
| 60 |
+
unchanged Jev client interface into the two Laya providers only. No model
|
| 61 |
+
weights or ML dependencies are installed in Opportunity. Remove all four
|
| 62 |
+
variables to restore the old Jev provider chain without changing its YAML
|
| 63 |
+
questions, thresholds, or credentials.
|
app.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU-only, single-checkpoint deployment of Laya's native Jev HTTP API."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
MODEL = "convaiinnovations/laya-multilingual"
|
| 7 |
+
MODEL_REVISION = "1720e3e3357cfe1e281542e223f8273b0890ca34"
|
| 8 |
+
WORKERS = 4 if os.environ.get("SPACE_ID") else 1
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def jev_result(result, confidence):
|
| 12 |
+
"""Keep Jev's confidence semantics; incomplete input must never hard-reject."""
|
| 13 |
+
usage = result.get("usage") or {}
|
| 14 |
+
truncated = bool(usage.get("truncated") or usage.get("state_tokens_dropped"))
|
| 15 |
+
for answer in result["answers"].values():
|
| 16 |
+
if answer.get("type") in {"choice", "score"}:
|
| 17 |
+
value = confidence(answer)
|
| 18 |
+
answer["confidence"] = 0.0 if truncated or value is None else round(value, 4)
|
| 19 |
+
result["model"] = MODEL
|
| 20 |
+
return result
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def create_app():
|
| 24 |
+
if not os.environ.get("LAYA_API_KEY", "").strip():
|
| 25 |
+
raise RuntimeError("LAYA_API_KEY is required")
|
| 26 |
+
os.environ.update(
|
| 27 |
+
LAYA_DEVICE="cpu",
|
| 28 |
+
LAYA_MAX_CONCURRENT="8" if WORKERS == 4 else "16",
|
| 29 |
+
LAYA_JEV_STRICT="1",
|
| 30 |
+
LAYA_IDLE_UNLOAD_SECONDS="0",
|
| 31 |
+
LAYA_TOURNAMENT_GROUP="0",
|
| 32 |
+
LAYA_MAX_TOKEN_BUDGET="1024",
|
| 33 |
+
CUDA_VISIBLE_DEVICES="",
|
| 34 |
+
)
|
| 35 |
+
import torch
|
| 36 |
+
from laya.confidence import jev_confidence
|
| 37 |
+
from laya.router import Router
|
| 38 |
+
from laya.serve import create_app as native_app
|
| 39 |
+
|
| 40 |
+
torch.set_num_threads(max(1, min(4, int(os.environ.get("LAYA_THREADS", "2")))))
|
| 41 |
+
|
| 42 |
+
class MultilingualRouter(Router):
|
| 43 |
+
def resolve(self, name):
|
| 44 |
+
if name == "typesafe-ai/jev":
|
| 45 |
+
name = "multilingual"
|
| 46 |
+
return super().resolve(name)
|
| 47 |
+
|
| 48 |
+
def route(self, state, questions=None, **kwargs):
|
| 49 |
+
return super().route(state, questions, model="multilingual")
|
| 50 |
+
|
| 51 |
+
def predict(self, state, questions, **kwargs):
|
| 52 |
+
return jev_result(super().predict(state, questions, **kwargs), jev_confidence)
|
| 53 |
+
|
| 54 |
+
router = MultilingualRouter(
|
| 55 |
+
device="cpu", standalone_repos=True, max_loaded=1,
|
| 56 |
+
revisions={"multilingual": MODEL_REVISION},
|
| 57 |
+
)
|
| 58 |
+
# Removing the other registry entries also makes an accidental direct load fail closed.
|
| 59 |
+
router.models = {"multilingual": MODEL}
|
| 60 |
+
router.preload(["multilingual"])
|
| 61 |
+
app = native_app(router=router)
|
| 62 |
+
app.state.laya_router = router
|
| 63 |
+
return app
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
if __name__ == "__main__":
|
| 67 |
+
if os.environ.get("SPACES_ZERO_GPU", "").lower() in {"1", "t", "true"}:
|
| 68 |
+
import spaces
|
| 69 |
+
|
| 70 |
+
@spaces.GPU(duration=1)
|
| 71 |
+
def unused_gpu():
|
| 72 |
+
return True
|
| 73 |
+
|
| 74 |
+
# shortcut: spaces 0.51.3 only invokes this from Gradio.launch; recheck on SDK upgrades.
|
| 75 |
+
spaces.zero.startup()
|
| 76 |
+
print("ZeroGPU startup registered; dummy GPU function not invoked", flush=True)
|
| 77 |
+
|
| 78 |
+
import uvicorn
|
| 79 |
+
|
| 80 |
+
uvicorn.run("app:create_app", factory=True, host="0.0.0.0",
|
| 81 |
+
port=int(os.environ.get("PORT", "7860")), workers=WORKERS, access_log=False)
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch==2.8.0
|
| 2 |
+
transformers==5.20.2
|
| 3 |
+
huggingface-hub==1.33.0
|
| 4 |
+
laya[serve] @ https://github.com/nandhakishorm/laya/archive/5a6dc4b809659dab3b9b3062f5e3902db983818a.tar.gz
|