"""Latency probe — isolate WHERE the agent's response delay comes from. Measures three things, independently of the voice loop: 1. Round-trip time (RTT) to api.elevenlabs.io and openrouter.ai — a cheap proxy for "your internet to each service". 2. The OpenRouter/DeepSeek LLM call, STREAMING, reporting time-to-first-token (TTFT, what you actually feel) vs total time vs tokens produced. 3. Variance across N runs (min/median/max), so spikes show up. Run: python -m scripts.latency_probe (from the backend/ dir) python -m scripts.latency_probe --runs 8 --words 40 Interpretation: * LLM TTFT spikes to several seconds -> it's the model/OpenRouter routing, NOT your home internet. A faster Wi-Fi won't help. * Total grows with token count -> generation length is the cost; cap max_tokens + prompt for short replies + ensure streaming. * Only the ElevenLabs RTT is high -> then your local network to the audio service is implicated. """ from __future__ import annotations import argparse import asyncio import statistics import time import httpx # Load .env so OPENROUTER_API_KEY is visible when run standalone. from dotenv import load_dotenv load_dotenv(override=True) # .env wins over any stale shell var from app.config import get_settings # noqa: E402 async def rtt(url: str, n: int = 5) -> list[float]: """Crude RTT: time a tiny GET to the host n times (ms).""" out: list[float] = [] async with httpx.AsyncClient(timeout=10) as client: for _ in range(n): t0 = time.perf_counter() try: await client.get(url) except Exception: # Even a 4xx/redirect is fine; we only care about the round trip. pass out.append((time.perf_counter() - t0) * 1000) return out async def llm_once(prompt: str) -> tuple[float, float, int, str]: """One streaming completion. Returns (ttft_s, total_s, token_chunks, provider). Uses the SAME routing the agent uses (routed_model + provider prefs) so the numbers reflect production. 'provider' is which OpenRouter upstream served it. """ import json as _json from app.services import llm_client as L settings = get_settings() headers = { "Authorization": f"Bearer {settings.OPENROUTER_API_KEY}", "Content-Type": "application/json", } payload: dict = { "model": L.routed_model(), "messages": [ {"role": "system", "content": "You are a person on a phone call. Reply briefly."}, {"role": "user", "content": prompt}, ], "stream": True, "temperature": 0.8, "max_tokens": settings.LLM_MAX_TOKENS, } prefs = L.provider_preferences() if prefs: payload["provider"] = prefs url = f"{settings.OPENROUTER_BASE_URL.rstrip('/')}/chat/completions" t0 = time.perf_counter() ttft = -1.0 chunks = 0 provider = "?" async with httpx.AsyncClient(timeout=60) as client: async with client.stream("POST", url, headers=headers, json=payload) as resp: resp.raise_for_status() async for line in resp.aiter_lines(): if not line or not line.startswith("data:"): continue if line.strip() == "data: [DONE]": break if provider == "?": try: # OpenRouter reports the upstream provider in the chunks provider = _json.loads(line[5:]).get("provider", "?") or "?" except Exception: pass if ttft < 0: ttft = time.perf_counter() - t0 # first token arrived chunks += 1 total = time.perf_counter() - t0 return ttft, total, chunks, provider def summarize(label: str, values: list[float], unit: str) -> None: if not values: print(f" {label}: (no data)") return print( f" {label}: min {min(values):.0f}{unit} " f"median {statistics.median(values):.0f}{unit} " f"max {max(values):.0f}{unit}" ) async def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--runs", type=int, default=6) ap.add_argument("--words", type=int, default=25) ap.add_argument( "--sort", default=None, help="Override LLM_PROVIDER_SORT for this run: throughput|price|latency|''", ) ap.add_argument( "--order", default=None, help="Override LLM_PROVIDER_ORDER, comma-separated (e.g. deepinfra,novita)", ) ap.add_argument( "--no-fallbacks", action="store_true", help="Pin EXCLUSIVELY to --order providers (no routing to anyone else)", ) args = ap.parse_args() settings = get_settings() # A/B routing without editing .env — mutate the in-memory settings. if args.sort is not None: settings.LLM_PROVIDER_SORT = args.sort.strip().lower() if args.order is not None: settings.LLM_PROVIDER_ORDER = [p.strip() for p in args.order.split(",") if p.strip()] if args.no_fallbacks: settings.LLM_ALLOW_FALLBACKS = False from app.services import llm_client as L print(f"Model: {L.routed_model()}") print(f"Sort: {settings.LLM_PROVIDER_SORT or '(default)'} " f"Order: {settings.LLM_PROVIDER_ORDER or '(none)'}") print(f"Base: {settings.OPENROUTER_BASE_URL}\n") # --- 1. RTT to the two relevant hosts (your internet) ------------------- print("RTT (your machine -> service):") el = await rtt("https://api.elevenlabs.io/v1/health") orr = await rtt("https://openrouter.ai/api/v1/models") summarize("ElevenLabs", el, "ms") summarize("OpenRouter", orr, "ms") print() # --- 2 & 3. LLM TTFT / total over N runs -------------------------------- if not settings.llm_configured: print("OPENROUTER_API_KEY is blank — skipping LLM timing. Set it in .env.") return prompt = ( f"In about {args.words} words, react as a skeptical Austrian prospect " "to a financial advisor's opening." ) ttfts: list[float] = [] totals: list[float] = [] providers: set[str] = set() print(f"LLM streaming, {args.runs} runs:") for i in range(args.runs): try: ttft, total, chunks, provider = await llm_once(prompt) except Exception as exc: # noqa: BLE001 print(f" run {i + 1}: ERROR {exc}") continue providers.add(provider) ttfts.append(ttft * 1000) totals.append(total * 1000) print( f" run {i + 1}: TTFT {ttft * 1000:.0f}ms " f"total {total * 1000:.0f}ms chunks {chunks} via {provider}" ) print() summarize("TTFT (time to first token)", ttfts, "ms") summarize("Total completion", totals, "ms") print(f" Providers seen: {', '.join(sorted(providers)) or '?'}") if ttfts: budget = "OK for 1-2s" if statistics.median(ttfts) <= 800 else "TOO SLOW for 1-2s" print(f" TTFT budget check (<=800ms): {budget}") if __name__ == "__main__": asyncio.run(main())