Spaces:
Running
Running
Download returns/app/availability.py from vllm-sr/System-Duo-Demos: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/spaces/vllm-sr/System-Duo-Demos/resolve/main/returns/app/availability.py
- Command line
-
hf download hf://spaces/vllm-sr/System-Duo-Demos/returns/app/availability.py
-
curl -L -o availability.py https://huggingface.co/spaces/vllm-sr/System-Duo-Demos/resolve/main/returns/app/availability.py
10.6 kB
| """Which models are up: a tracker per model, fed by a background probe and by every real call (a circuit breaker). | |
| Models: "d3" (D3 on /v1/systemone), "guard" (the Vela guardrail on /v1/systemone) and "llm" (vllm-sr/auto on | |
| /v1/chat/completions), all on the hosted vllm-sr API. | |
| - Every real call goes through `call(name)`: a connection error, a timeout, a 5xx or an unreadable answer is a | |
| failure; FAILS failures in a row mark the model down for BREAK_S seconds, and calls to it are then refused at once | |
| (Unavailable, no request sent). One success marks it up again. 429s and full slots (limits.Busy) are capacity, not | |
| failures. | |
| - `start_probes()` checks the API every HEALTH_INTERVAL seconds with one GET /v1/models (public: no key, no model | |
| call, no quota): the API answers and lists the configured models. A model that is down is probed again when its breaker | |
| ends. Probes do not take model-call slots; real calls keep the tracker current in between. | |
| - The flow asks `available(name)` before it relies on a model; the reasons the customer can see are plain words | |
| ("unavailable", "timed out"); exception text only goes to the server log. | |
| """ | |
| import os | |
| import threading | |
| import time | |
| from contextlib import contextmanager | |
| import requests | |
| from keys import KeyFailed | |
| from limits import Busy | |
| NAMES = ("d3", "guard", "llm") | |
| LABEL = {"d3": "D3", "guard": "Vela guardrail", "llm": "Language model"} | |
| FAILS = 3 | |
| BREAK_S = float(os.environ.get("BREAKER_S", "30")) | |
| HEALTH_INTERVAL = max(10.0, float(os.environ.get("HEALTH_INTERVAL", "60"))) | |
| # per call, seconds (a streamed reply: the whole stream); the hosted API stops a request at 35 s | |
| TIMEOUT = {"d3": min(30.0, float(os.environ.get("D3_TIMEOUT_S", "15"))), | |
| "guard": min(30.0, float(os.environ.get("GUARD_TIMEOUT_S", "5"))), | |
| "llm": min(34.0, float(os.environ.get("LLM_TIMEOUT_S", "30"))), | |
| "probe": 5.0} | |
| UNAVAILABLE_TEXT = "The assistant is unavailable right now. Please try again in a minute." | |
| def timeout(name): | |
| """(connect, read) for requests.""" | |
| t = TIMEOUT[name] | |
| return min(3.0, t), t | |
| class Unavailable(Exception): | |
| """A model call that failed or was not sent because the model is down. `reason`: "unavailable" or "timed out".""" | |
| def __init__(self, name, reason="unavailable"): | |
| super().__init__(f"{LABEL.get(name, name)} {reason}") | |
| self.name, self.reason = name, reason | |
| def text(self): | |
| """One plain line for a flow step.""" | |
| return f"{LABEL.get(self.name, self.name)} {self.reason}" | |
| def log(msg): | |
| print(f"models: {msg}", flush=True) | |
| class Tracker: | |
| def __init__(self, fails=FAILS, break_s=BREAK_S): | |
| self.lock = threading.Lock() | |
| self.fails, self.break_s = fails, break_s | |
| # up: None until the first probe or call; down_until: breaker open until then | |
| self.state = {n: {"up": None, "fails": 0, "down_until": 0.0, "reason": None, "checked": None, | |
| "next_probe": 0.0} for n in NAMES} | |
| def available(self, name, now=None): | |
| """False while the model's breaker is open. Unknown (not checked yet) counts as available.""" | |
| now = time.time() if now is None else now | |
| with self.lock: | |
| return self.state[name]["down_until"] <= now | |
| def success(self, name, now=None): | |
| now = time.time() if now is None else now | |
| with self.lock: | |
| s = self.state[name] | |
| was = s["up"] | |
| s.update(up=True, fails=0, down_until=0.0, reason=None, checked=now) | |
| if was is False: | |
| log(f"{name} up") | |
| def failure(self, name, reason, detail="", now=None, probe=False): | |
| """A failed call (or probe: down at once). Returns True when this opened the breaker.""" | |
| now = time.time() if now is None else now | |
| with self.lock: | |
| s = self.state[name] | |
| s["fails"] = self.fails if probe else s["fails"] + 1 | |
| s["checked"] = now | |
| opened = s["fails"] >= self.fails | |
| if opened: | |
| was_down = s["down_until"] > now | |
| s.update(up=False, down_until=now + self.break_s, reason=reason, next_probe=now + self.break_s) | |
| log(f"{name} {'probe' if probe else 'call'} failed ({reason}; {s['fails']} in a row): {detail[:300]}") | |
| if opened and not was_down: | |
| log(f"{name} down for {self.break_s:.0f} s") | |
| return opened | |
| def snapshot(self, now=None): | |
| now = time.time() if now is None else now | |
| with self.lock: | |
| out = {} | |
| for n, s in self.state.items(): | |
| down = s["down_until"] > now | |
| out[n] = {"state": "down" if down else "unknown" if s["up"] is None else "up", | |
| "reason": s["reason"] if down else None, | |
| "retry_in_s": round(s["down_until"] - now) if down else None, | |
| "checked_s_ago": None if s["checked"] is None else round(now - s["checked"])} | |
| return out | |
| def reset(self): | |
| with self.lock: | |
| for s in self.state.values(): | |
| s.update(up=None, fails=0, down_until=0.0, reason=None, checked=None, next_probe=0.0) | |
| TRACKER = Tracker() | |
| available = TRACKER.available | |
| def classify(e): | |
| """(reason, counts as a failure) for an exception from a model call.""" | |
| if isinstance(e, requests.Timeout): | |
| return "timed out", True | |
| if isinstance(e, requests.ConnectionError): | |
| # a read timeout in the middle of a stream surfaces as a ConnectionError | |
| return ("timed out" if "timed out" in str(e).lower() else "unavailable"), True | |
| if isinstance(e, requests.HTTPError): | |
| code = e.response.status_code if e.response is not None else 500 | |
| return "unavailable", code >= 500 | |
| if isinstance(e, (ValueError, KeyError, IndexError, TypeError, AttributeError)): # not the answer we expect | |
| return "unavailable", True | |
| return "unavailable", False | |
| def call(name): | |
| """Wrap one real model call: refused at once while the model is down; failures and successes feed the breaker; | |
| every failure leaves as Unavailable (Busy passes through: capacity is not a failure, except a 503 twice).""" | |
| if not TRACKER.available(name): | |
| raise Unavailable(name, "unavailable") | |
| try: | |
| yield | |
| except KeyFailed: # the visitor's key, not the model: not a failure, no template reply | |
| raise | |
| except Unavailable as e: # raised by the call itself (a stream over its time limit) | |
| TRACKER.failure(name, e.reason, str(e)) | |
| raise | |
| except Busy as e: | |
| if getattr(e, "code", None) == 503: # 503 twice: the Router has no healthy backend, or it is overloaded | |
| TRACKER.failure(name, "unavailable", str(e)) | |
| raise | |
| except Exception as e: # noqa: BLE001 | |
| reason, counts = classify(e) | |
| if counts: | |
| TRACKER.failure(name, reason, f"{type(e).__name__}: {e}") | |
| else: | |
| log(f"{name} call failed (not counted): {type(e).__name__}: {str(e)[:300]}") | |
| raise Unavailable(name, reason) from e | |
| TRACKER.success(name) | |
| # ---------------------------------------------------------------------------------------------- probes | |
| def model_ids(): | |
| import models | |
| return {"d3": models.D3_MODEL, "guard": models.GUARD_MODEL, "llm": models.LLM_MODEL} | |
| def probe_models(): | |
| """One GET /v1/models (public, no key: no model call, no quota): {name: (ok, reason, detail)} for every model. A model the API | |
| does not list is "not offered" (a wrong model name in the environment, or a key without access).""" | |
| import models | |
| try: | |
| r = requests.get(f"{models.API_BASE}/v1/models", timeout=timeout("probe")) # public: no visitor key needed | |
| if r.status_code == 429: | |
| return {n: (True, None, "429: up but throttled") for n in NAMES} | |
| if r.status_code in (401, 403): | |
| return {n: (False, "unavailable", f"API key refused ({r.status_code})") for n in NAMES} | |
| r.raise_for_status() | |
| listed = {m.get("id") for m in r.json().get("data") or []} | |
| except Exception as e: # noqa: BLE001 | |
| reason, _counts = classify(e) | |
| return {n: (False, reason, f"{type(e).__name__}") for n in NAMES} | |
| return {n: (True, None, "") if mid in listed else (False, "not offered", f"{mid} is not listed by /v1/models") | |
| for n, mid in model_ids().items()} | |
| def probe(name): | |
| """Returns (ok, reason, detail) for one model (one GET /v1/models).""" | |
| return probe_models()[name] | |
| def probe_all(names=NAMES): | |
| """Probe now: one GET /v1/models for all models.""" | |
| res = probe_models() | |
| for n in names: | |
| ok, reason, detail = res[n] | |
| if ok: | |
| TRACKER.success(n) | |
| else: | |
| TRACKER.failure(n, reason, detail, probe=True) | |
| with TRACKER.lock: | |
| s = TRACKER.state[n] | |
| if ok: | |
| s["next_probe"] = time.time() + HEALTH_INTERVAL | |
| def _loop(): | |
| while True: | |
| now = time.time() | |
| with TRACKER.lock: | |
| due = [n for n, s in TRACKER.state.items() if s["next_probe"] <= now] | |
| if due: | |
| probe_all(due) | |
| time.sleep(1) | |
| _started = threading.Lock() | |
| _running = [] | |
| def start_probes(): | |
| """Start the background probe thread once (first round at once).""" | |
| with _started: | |
| if not _running: | |
| t = threading.Thread(target=_loop, daemon=True, name="model-probes") | |
| t.start() | |
| _running.append(t) | |
| def status(snap=None): | |
| """What the top bar says: (key, plain words). Keys: ok, checking, guard_down, standard, unavailable.""" | |
| snap = snap or TRACKER.snapshot() | |
| down = {n for n, s in snap.items() if s["state"] == "down"} | |
| if "d3" in down: | |
| return "unavailable", "Assistant unavailable" | |
| if "llm" in down: | |
| return "standard", "Standard replies: language model unavailable" | |
| if "guard" in down: | |
| return "guard_down", "Vela guardrail unavailable: D3 checks messages alone" | |
| if any(s["state"] == "unknown" for s in snap.values()): | |
| return "checking", "Checking systems" | |
| return "ok", "All systems working" | |
| def health(): | |
| """GET /health and /api/health: the tracker's view (no probe of its own). ok: every model up.""" | |
| snap = TRACKER.snapshot() | |
| key, words = status(snap) | |
| return {"ok": all(s["state"] == "up" for s in snap.values()), "status": key, "message": words, | |
| **{n: s["state"] == "up" for n, s in snap.items()}, "models": snap, | |
| "probe_interval_s": HEALTH_INTERVAL} | |