Spaces:
Sleeping
Sleeping
| """Latency probe — isolate WHERE the agent's response delay comes from. | |
| Measures three things, independently of the voice loop: | |
| 1. Round-trip time (RTT) to api.elevenlabs.io and openrouter.ai — a cheap | |
| proxy for "your internet to each service". | |
| 2. The OpenRouter/DeepSeek LLM call, STREAMING, reporting time-to-first-token | |
| (TTFT, what you actually feel) vs total time vs tokens produced. | |
| 3. Variance across N runs (min/median/max), so spikes show up. | |
| Run: python -m scripts.latency_probe (from the backend/ dir) | |
| python -m scripts.latency_probe --runs 8 --words 40 | |
| Interpretation: | |
| * LLM TTFT spikes to several seconds -> it's the model/OpenRouter routing, | |
| NOT your home internet. A faster Wi-Fi won't help. | |
| * Total grows with token count -> generation length is the cost; cap | |
| max_tokens + prompt for short replies + ensure streaming. | |
| * Only the ElevenLabs RTT is high -> then your local network to the audio | |
| service is implicated. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import statistics | |
| import time | |
| import httpx | |
| # Load .env so OPENROUTER_API_KEY is visible when run standalone. | |
| from dotenv import load_dotenv | |
| load_dotenv(override=True) # .env wins over any stale shell var | |
| from app.config import get_settings # noqa: E402 | |
| async def rtt(url: str, n: int = 5) -> list[float]: | |
| """Crude RTT: time a tiny GET to the host n times (ms).""" | |
| out: list[float] = [] | |
| async with httpx.AsyncClient(timeout=10) as client: | |
| for _ in range(n): | |
| t0 = time.perf_counter() | |
| try: | |
| await client.get(url) | |
| except Exception: | |
| # Even a 4xx/redirect is fine; we only care about the round trip. | |
| pass | |
| out.append((time.perf_counter() - t0) * 1000) | |
| return out | |
| async def llm_once(prompt: str) -> tuple[float, float, int, str]: | |
| """One streaming completion. Returns (ttft_s, total_s, token_chunks, provider). | |
| Uses the SAME routing the agent uses (routed_model + provider prefs) so the | |
| numbers reflect production. 'provider' is which OpenRouter upstream served it. | |
| """ | |
| import json as _json | |
| from app.services import llm_client as L | |
| settings = get_settings() | |
| headers = { | |
| "Authorization": f"Bearer {settings.OPENROUTER_API_KEY}", | |
| "Content-Type": "application/json", | |
| } | |
| payload: dict = { | |
| "model": L.routed_model(), | |
| "messages": [ | |
| {"role": "system", "content": "You are a person on a phone call. Reply briefly."}, | |
| {"role": "user", "content": prompt}, | |
| ], | |
| "stream": True, | |
| "temperature": 0.8, | |
| "max_tokens": settings.LLM_MAX_TOKENS, | |
| } | |
| prefs = L.provider_preferences() | |
| if prefs: | |
| payload["provider"] = prefs | |
| url = f"{settings.OPENROUTER_BASE_URL.rstrip('/')}/chat/completions" | |
| t0 = time.perf_counter() | |
| ttft = -1.0 | |
| chunks = 0 | |
| provider = "?" | |
| async with httpx.AsyncClient(timeout=60) as client: | |
| async with client.stream("POST", url, headers=headers, json=payload) as resp: | |
| resp.raise_for_status() | |
| async for line in resp.aiter_lines(): | |
| if not line or not line.startswith("data:"): | |
| continue | |
| if line.strip() == "data: [DONE]": | |
| break | |
| if provider == "?": | |
| try: # OpenRouter reports the upstream provider in the chunks | |
| provider = _json.loads(line[5:]).get("provider", "?") or "?" | |
| except Exception: | |
| pass | |
| if ttft < 0: | |
| ttft = time.perf_counter() - t0 # first token arrived | |
| chunks += 1 | |
| total = time.perf_counter() - t0 | |
| return ttft, total, chunks, provider | |
| def summarize(label: str, values: list[float], unit: str) -> None: | |
| if not values: | |
| print(f" {label}: (no data)") | |
| return | |
| print( | |
| f" {label}: min {min(values):.0f}{unit} " | |
| f"median {statistics.median(values):.0f}{unit} " | |
| f"max {max(values):.0f}{unit}" | |
| ) | |
| async def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--runs", type=int, default=6) | |
| ap.add_argument("--words", type=int, default=25) | |
| ap.add_argument( | |
| "--sort", | |
| default=None, | |
| help="Override LLM_PROVIDER_SORT for this run: throughput|price|latency|''", | |
| ) | |
| ap.add_argument( | |
| "--order", | |
| default=None, | |
| help="Override LLM_PROVIDER_ORDER, comma-separated (e.g. deepinfra,novita)", | |
| ) | |
| ap.add_argument( | |
| "--no-fallbacks", | |
| action="store_true", | |
| help="Pin EXCLUSIVELY to --order providers (no routing to anyone else)", | |
| ) | |
| args = ap.parse_args() | |
| settings = get_settings() | |
| # A/B routing without editing .env — mutate the in-memory settings. | |
| if args.sort is not None: | |
| settings.LLM_PROVIDER_SORT = args.sort.strip().lower() | |
| if args.order is not None: | |
| settings.LLM_PROVIDER_ORDER = [p.strip() for p in args.order.split(",") if p.strip()] | |
| if args.no_fallbacks: | |
| settings.LLM_ALLOW_FALLBACKS = False | |
| from app.services import llm_client as L | |
| print(f"Model: {L.routed_model()}") | |
| print(f"Sort: {settings.LLM_PROVIDER_SORT or '(default)'} " | |
| f"Order: {settings.LLM_PROVIDER_ORDER or '(none)'}") | |
| print(f"Base: {settings.OPENROUTER_BASE_URL}\n") | |
| # --- 1. RTT to the two relevant hosts (your internet) ------------------- | |
| print("RTT (your machine -> service):") | |
| el = await rtt("https://api.elevenlabs.io/v1/health") | |
| orr = await rtt("https://openrouter.ai/api/v1/models") | |
| summarize("ElevenLabs", el, "ms") | |
| summarize("OpenRouter", orr, "ms") | |
| print() | |
| # --- 2 & 3. LLM TTFT / total over N runs -------------------------------- | |
| if not settings.llm_configured: | |
| print("OPENROUTER_API_KEY is blank — skipping LLM timing. Set it in .env.") | |
| return | |
| prompt = ( | |
| f"In about {args.words} words, react as a skeptical Austrian prospect " | |
| "to a financial advisor's opening." | |
| ) | |
| ttfts: list[float] = [] | |
| totals: list[float] = [] | |
| providers: set[str] = set() | |
| print(f"LLM streaming, {args.runs} runs:") | |
| for i in range(args.runs): | |
| try: | |
| ttft, total, chunks, provider = await llm_once(prompt) | |
| except Exception as exc: # noqa: BLE001 | |
| print(f" run {i + 1}: ERROR {exc}") | |
| continue | |
| providers.add(provider) | |
| ttfts.append(ttft * 1000) | |
| totals.append(total * 1000) | |
| print( | |
| f" run {i + 1}: TTFT {ttft * 1000:.0f}ms " | |
| f"total {total * 1000:.0f}ms chunks {chunks} via {provider}" | |
| ) | |
| print() | |
| summarize("TTFT (time to first token)", ttfts, "ms") | |
| summarize("Total completion", totals, "ms") | |
| print(f" Providers seen: {', '.join(sorted(providers)) or '?'}") | |
| if ttfts: | |
| budget = "OK for 1-2s" if statistics.median(ttfts) <= 800 else "TOO SLOW for 1-2s" | |
| print(f" TTFT budget check (<=800ms): {budget}") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |