yapapa / tests /test_admin_api.py
wiwi deploy
Deploy wiwi 13f8ec7
928fda2
Raw History Blame Contribute Delete
25.3 kB
"""Admin API surface: new endpoints, guards, live-pool mutations, stats math."""
from __future__ import annotations
import os
from pathlib import Path
import httpx
import orjson
import pytest
import respx
from asgi_lifespan import LifespanManager
import wiwi.server.app as app_mod
from wiwi.config import (
DeploymentParams,
GeneralSettings,
KeyDef,
ModelEntry,
ProviderDef,
WiwiConfig,
)
from wiwi.logging_core.events import LogEvent
from wiwi.server.stats import overview, timeseries
MASTER = "sk-wiwi-master-test"
AUTH = {"Authorization": f"Bearer {MASTER}"}
def _config() -> WiwiConfig:
return WiwiConfig(
providers=[ProviderDef(name="p1", provider="openai",
keys=[KeyDef(label="a", key="sk-test-key-abcdef123456")])],
model_list=[ModelEntry(model_name="gpt-4o",
wiwi_params=DeploymentParams(provider="p1",
model="gpt-4o"))],
general_settings=GeneralSettings(master_key=MASTER,
database_url="sqlite+aiosqlite:///:memory:"),
)
@pytest.fixture
async def client():
app = app_mod.create_app(_config())
async with LifespanManager(app):
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport,
base_url="http://test") as c:
yield c
# -- auth guards ---------------------------------------------------------------
@pytest.mark.parametrize("method,path", [
("GET", "/admin/providers"),
("PATCH", "/admin/providers/p1/keys/a"),
("POST", "/admin/providers/p1/keys"),
("POST", "/admin/providers"),
("GET", "/admin/models"),
("PATCH", "/admin/model-groups/gpt-4o"),
("PATCH", "/admin/keys/k123"),
("GET", "/admin/stats/overview"),
("GET", "/admin/stats/timeseries"),
("GET", "/admin/logs/proxy"),
("GET", "/admin/alert-rules"),
("PUT", "/admin/alert-rules"),
])
async def test_new_endpoints_require_master(client, method, path):
r = await client.request(method, path, json={"weight": 1})
assert r.status_code == 401
r = await client.request(method, path, json={"weight": 1},
headers={"Authorization": "Bearer wrong"})
assert r.status_code == 401
# -- providers -------------------------------------------------------------------
async def test_providers_lists_pool_without_secrets(client):
r = await client.get("/admin/providers", headers=AUTH)
assert r.status_code == 200
providers = r.json()["providers"]
assert len(providers) == 1
p = providers[0]
assert p["name"] == "p1" and p["provider_type"] == "openai"
assert p["healthy"] is True
key = p["keys"][0]
assert key["label"] == "a"
assert key["status"] == "active"
assert key["req_count"] == 0 and key["err_count"] == 0
assert key["last_used_ts"] is None
# H4: plaintext secrets must NOT be exposed in list responses — the
# masked form is sufficient for the admin UI; reveal happens via the
# dedicated POST /admin/providers/{name}/keys/{label}/reveal endpoint.
assert "secret" not in key
assert key["masked"].startswith("sk-te")
async def test_patch_provider_key_updates_live_pool(client):
r = await client.patch("/admin/providers/p1/keys/a",
json={"enabled": False, "weight": 5}, headers=AUTH)
assert r.status_code == 200
key = r.json()["key"]
assert key["enabled"] is False and key["weight"] == 5
assert key["status"] == "disabled"
# visible in the listing too
lst = (await client.get("/admin/providers", headers=AUTH)).json()
k = lst["providers"][0]["keys"][0]
assert k["enabled"] is False and k["weight"] == 5
bad = await client.patch("/admin/providers/nope/keys/a",
json={"weight": 2}, headers=AUTH)
assert bad.status_code == 404
bad2 = await client.patch("/admin/providers/p1/keys/zzz",
json={"weight": 2}, headers=AUTH)
assert bad2.status_code == 404
bad3 = await client.patch("/admin/providers/p1/keys/a",
json={"weight": "lots"}, headers=AUTH)
assert bad3.status_code == 400
async def test_add_provider_key_runtime(client):
r = await client.post("/admin/providers/p1/keys",
json={"label": "backup", "key": "sk-second-key-xyz",
"weight": 2}, headers=AUTH)
assert r.status_code == 200
dup = await client.post("/admin/providers/p1/keys",
json={"label": "backup", "key": "x"}, headers=AUTH)
assert dup.status_code == 409
missing = await client.post("/admin/providers/nope/keys",
json={"label": "b", "key": "k"}, headers=AUTH)
assert missing.status_code == 404
provs = (await client.get("/admin/providers", headers=AUTH)).json()
labels = [k["label"] for k in provs["providers"][0]["keys"]]
assert labels == ["a", "backup"]
# H4: list responses must never include plaintext secrets.
provs2 = (await client.get("/admin/providers", headers=AUTH)).json()
for k in provs2["providers"][0]["keys"]:
assert "secret" not in k
assert "sk-second-key-xyz" not in (await client.get(
"/admin/providers", headers=AUTH)).text
async def test_add_provider_account_runtime(client):
r = await client.post("/admin/providers",
json={"name": "p2", "provider_type": "openai-compatible",
"base_url": "https://relay.example/v1",
"label": "main", "key": "sk-new-acct-key"}, headers=AUTH)
assert r.status_code == 200
dup = await client.post("/admin/providers",
json={"name": "p2", "key": "k"}, headers=AUTH)
assert dup.status_code == 409
no_key = await client.post("/admin/providers",
json={"name": "p3"}, headers=AUTH)
assert no_key.status_code == 400
provs = (await client.get("/admin/providers", headers=AUTH)).json()
names = [p["name"] for p in provs["providers"]]
assert names == ["p1", "p2"]
p2 = provs["providers"][1]
assert p2["base_url"] == "https://relay.example/v1"
# -- models & routing -------------------------------------------------------------
async def test_models_listing_shape(client):
r = await client.get("/admin/models", headers=AUTH)
assert r.status_code == 200
data = r.json()
assert data["strategy"] == "simple-shuffle"
g = data["groups"][0]
assert g["name"] == "gpt-4o"
d = g["deployments"][0]
assert d["provider"] == "p1" and d["model_id"] == "gpt-4o"
assert d["weight"] == 1 and d["available"] is True
async def test_patch_model_group_weights_and_strategy(client):
r = await client.patch("/admin/model-groups/gpt-4o",
json={"weights": {"p1/gpt-4o": 7},
"strategy": "latency-based"}, headers=AUTH)
assert r.status_code == 200
models = (await client.get("/admin/models", headers=AUTH)).json()
assert models["strategy"] == "latency-based"
assert models["groups"][0]["deployments"][0]["weight"] == 7
# patch via alias name resolves to the same group
cfg = _config()
cfg.router_settings.model_group_alias = {"gpt4": "gpt-4o"}
app = app_mod.create_app(cfg)
async with LifespanManager(app):
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport,
base_url="http://test") as c:
r2 = await c.patch("/admin/model-groups/gpt4",
json={"weights": {"p1/gpt-4o": 3}}, headers=AUTH)
assert r2.status_code == 200
assert r2.json()["group"] == "gpt-4o"
for payload, code in [
({"strategy": "bogus"}, 400),
({"weights": {"other/model": 2}}, 400),
]:
rr = await client.patch("/admin/model-groups/gpt-4o", json=payload,
headers=AUTH)
assert rr.status_code == code
nf = await client.patch("/admin/model-groups/nope", json={}, headers=AUTH)
assert nf.status_code == 404
# -- provider model fetch + deployment create ---------------------------------------
@respx.mock
async def test_provider_models_fetch_openai(client):
respx.get("https://api.openai.com/v1/models").respond(
json={"object": "list",
"data": [{"id": "gpt-4o-mini"}, {"id": "gpt-4o"}, {"id": "o3"}]})
r = await client.get("/admin/providers/p1/models", headers=AUTH)
assert r.status_code == 200
assert r.json()["models"] == [{"id": "gpt-4o"}, {"id": "gpt-4o-mini"}, {"id": "o3"}]
@respx.mock
async def test_provider_models_upstream_error_passthrough(client):
respx.get("https://api.openai.com/v1/models").respond(
status_code=401, json={"error": {"message": "bad key"}})
r = await client.get("/admin/providers/p1/models", headers=AUTH)
assert r.status_code == 401
assert "bad key" in r.json()["error"]["message"]
async def test_provider_models_unknown_provider(client):
r = await client.get("/admin/providers/nope/models", headers=AUTH)
assert r.status_code == 404
def test_parse_models_response_shapes():
openai = app_mod._parse_models_response("openai", orjson.dumps(
{"data": [{"id": "b"}, {"id": "a"}, {"nope": 1}]}))
assert openai == [{"id": "b"}, {"id": "a"}]
gemini = app_mod._parse_models_response("gemini", orjson.dumps(
{"models": [{"name": "models/gemini-2.0-flash"},
{"name": "tunedModels/x", "supported": True}]}))
assert gemini == [{"id": "gemini-2.0-flash"}, {"id": "x"}]
assert app_mod._parse_models_response("openai", b"null") == []
async def test_add_deployment_runtime(client):
r = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "p1", "model_id": "gpt-4o-mini",
"weight": 3}, headers=AUTH)
assert r.status_code == 201
dep = r.json()["deployment"]
assert dep["provider"] == "p1" and dep["model_id"] == "gpt-4o-mini"
assert dep["weight"] == 3 and dep["available"] is True
# visible in the group listing
groups = (await client.get("/admin/models", headers=AUTH)).json()["groups"]
g = next(g for g in groups if g["name"] == "newgrp")
assert g["deployments"][0]["model_id"] == "gpt-4o-mini"
# second model on the same group; then duplicate is rejected
ok = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "p1", "model_id": "o3"}, headers=AUTH)
assert ok.status_code == 201
dup = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "p1", "model_id": "gpt-4o-mini"},
headers=AUTH)
assert dup.status_code == 409
# validation
missing = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "p1"}, headers=AUTH)
assert missing.status_code == 400
unknown = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "ghost", "model_id": "m"},
headers=AUTH)
assert unknown.status_code == 404
badw = await client.post("/admin/model-groups/newgrp/deployments",
json={"provider": "p1", "model_id": "m2",
"weight": "lots"}, headers=AUTH)
assert badw.status_code == 400
async def test_add_deployment_rejects_whitespace_model_id(client):
# Ids are typed by hand in the provider-detail UI; an embedded space is
# always a typo and upstreams reject it with a confusing 404.
r = await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "gpt 4o"},
headers=AUTH)
assert r.status_code == 400
assert "whitespace" in r.json()["error"]["message"]
# Surrounding whitespace is trimmed rather than rejected.
ok = await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": " gpt-4o "},
headers=AUTH)
assert ok.status_code == 201
assert ok.json()["deployment"]["model_id"] == "gpt-4o"
async def test_add_deployment_model_id_with_slash(client):
# Slashed ids (z-ai/glm-5.2) must survive; the group is the id itself.
r = await client.post("/admin/model-groups/z-ai/glm-5.2/deployments",
json={"provider": "p1", "model_id": "z-ai/glm-5.2"},
headers=AUTH)
assert r.status_code == 201
groups = (await client.get("/admin/models", headers=AUTH)).json()["groups"]
g = next(g for g in groups if g["name"] == "z-ai/glm-5.2")
assert g["deployments"][0]["model_id"] == "z-ai/glm-5.2"
async def test_delete_deployment_removes_only_target(client):
await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "a"}, headers=AUTH)
await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "b"}, headers=AUTH)
r = await client.delete("/admin/model-groups/grp/deployments"
"?provider=p1&model_id=a", headers=AUTH)
assert r.status_code == 200
assert r.json()["deleted"] is True
assert r.json()["group_emptied"] is False
g = next(g for g in (await client.get("/admin/models",
headers=AUTH)).json()["groups"]
if g["name"] == "grp")
assert sorted(d["model_id"] for d in g["deployments"]) == ["b"]
async def test_delete_deployment_pops_emptied_group(client):
await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "solo"}, headers=AUTH)
r = await client.delete("/admin/model-groups/grp/deployments"
"?provider=p1&model_id=solo", headers=AUTH)
assert r.status_code == 200
assert r.json()["group_emptied"] is True
names = [g["name"] for g in (await client.get("/admin/models",
headers=AUTH)).json()["groups"]]
assert "grp" not in names
async def test_delete_deployment_slashed_model_id(client):
# The target travels as query params precisely so slashed ids survive.
await client.post("/admin/model-groups/z-ai/glm-5.2/deployments",
json={"provider": "p1", "model_id": "z-ai/glm-5.2"},
headers=AUTH)
r = await client.delete("/admin/model-groups/z-ai/glm-5.2/deployments"
"?provider=p1&model_id=z-ai%2Fglm-5.2", headers=AUTH)
assert r.status_code == 200
assert r.json()["model_id"] == "z-ai/glm-5.2"
names = [g["name"] for g in (await client.get("/admin/models",
headers=AUTH)).json()["groups"]]
assert "z-ai/glm-5.2" not in names
async def test_delete_deployment_errors(client):
unknown_group = await client.delete(
"/admin/model-groups/nope/deployments?provider=p1&model_id=a",
headers=AUTH)
assert unknown_group.status_code == 404
# group exists, deployment does not
await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "a"}, headers=AUTH)
missing = await client.delete("/admin/model-groups/grp/deployments"
"?provider=p1&model_id=ghost", headers=AUTH)
assert missing.status_code == 404
wrong_provider = await client.delete("/admin/model-groups/grp/deployments"
"?provider=ghost&model_id=a", headers=AUTH)
assert wrong_provider.status_code == 404
no_params = await client.delete("/admin/model-groups/grp/deployments",
headers=AUTH)
assert no_params.status_code == 400
async def test_delete_deployment_requires_master(client):
await client.post("/admin/model-groups/grp/deployments",
json={"provider": "p1", "model_id": "a"}, headers=AUTH)
r = await client.delete("/admin/model-groups/grp/deployments"
"?provider=p1&model_id=a")
assert r.status_code == 401
g = next(g for g in (await client.get("/admin/models",
headers=AUTH)).json()["groups"]
if g["name"] == "grp")
assert [d["model_id"] for d in g["deployments"]] == ["a"]
# -- virtual keys PATCH ------------------------------------------------------------
async def test_patch_virtual_key_fields(client):
gen = await client.post("/admin/keys/generate",
json={"name": "team-x", "max_budget": 10.0}, headers=AUTH)
kid = gen.json()["id"]
r = await client.patch(f"/admin/keys/{kid}",
json={"max_budget": 25.5, "rpm": 60, "tpm": None,
"models": ["gpt-4o"], "expires_at": None},
headers=AUTH)
assert r.status_code == 200
key = r.json()["key"]
assert key["max_budget"] == 25.5
assert key["rpm"] == 60
assert key["tpm"] is None
assert key["models"] == ["gpt-4o"]
assert key["expires_at"] is None
# absent fields stay untouched
r2 = await client.patch(f"/admin/keys/{kid}", json={"rpm": 120}, headers=AUTH)
assert r2.json()["key"]["max_budget"] == 25.5
nf = await client.patch("/admin/keys/kzzz", json={"rpm": 5}, headers=AUTH)
assert nf.status_code == 404
# -- stats math (synthetic rings) ----------------------------------------------------
def _evt(ts: float, **kw) -> LogEvent:
defaults = {"stream": "request", "ts": ts}
defaults.update(kw)
return LogEvent(**defaults)
def test_overview_math():
now = 1_800_000_000.0
events = [
_evt(now - 30, status=200, tok_in=100, tok_cached=40, tok_reasoning=10,
tok_out=50, tps=80.0, ttft_ms=250.0, latency_ms=1000.0, cost=0.01,
cache_hit=True, cache_savings=0.002),
_evt(now - 20, status=200, tok_in=200, tok_out=100, tps=40.0, ttft_ms=500.0,
latency_ms=2000.0, cost=0.02),
_evt(now - 10, status=500, error_code="api_error", tok_in=50, cost=0.0),
_evt(now - 9999, status=200, tok_in=9_999), # outside window
]
ov = overview(events, minutes=5, now=now)
assert ov["requests"] == 3
assert ov["errors"] == 1
assert abs(ov["error_rate"] - round(1 / 3, 4)) < 1e-6
assert ov["tok_in"] == 350 and ov["tok_cached"] == 40
assert ov["tok_reasoning"] == 10 and ov["tok_out"] == 150
assert ov["cache_hits"] == 1
assert abs(ov["cache_hit_rate"] - round(1 / 3, 4)) < 1e-6
assert ov["tps_avg"] == 60.0 # (80 + 40) / 2, zero-tps excluded
assert ov["tps_p95"] == 80.0
assert ov["ttft_p95_ms"] == 500.0
assert ov["requests_per_minute"] == 0.6
assert abs(ov["cost"] - 0.03) < 1e-9
assert abs(ov["cache_savings"] - 0.002) < 1e-9
def test_timeseries_buckets():
now = (1_800_001_230 // 60) * 60 + 30 # mid-minute now
events = [
_evt(now - 95, tok_in=10, tok_out=5, tps=10.0),
_evt(now - 35, tok_in=20, tok_cached=8, tok_reasoning=2, tok_out=7, tps=30.0),
_evt(now - 25, tok_in=1, tok_out=1, tps=90.0),
_evt(now - 500, tok_in=99), # outside window
]
ts = timeseries(events, "minute", "tokens", 3, now=now)
assert ts["bucket_seconds"] == 60
buckets = ts["buckets"]
assert len(buckets) == 3
by_t = {b["t"]: b for b in buckets}
cur = by_t[int(now // 60) * 60] # holds now-25 event
mid = by_t[int(now // 60) * 60 - 60] # holds now-35 event
old = by_t[int(now // 60) * 60 - 120] # holds now-95 event
assert (cur["tok_in"], cur["tok_cached"], cur["tok_reasoning"],
cur["tok_out"]) == (1, 0, 0, 1)
assert (mid["tok_in"], mid["tok_cached"], mid["tok_reasoning"],
mid["tok_out"]) == (20, 8, 2, 7)
assert (old["tok_in"], old["tok_out"]) == (10, 5)
tp = timeseries(events, "minute", "tps", 3, now=now)
by_tp = {b["t"]: b for b in tp["buckets"]}
assert by_tp[int(now // 60) * 60]["tps_avg"] == 90.0
assert by_tp[int(now // 60) * 60 - 60]["tps_avg"] == 30.0
assert by_tp[int(now // 60) * 60 - 120]["tps_p95"] == 10.0
with pytest.raises(ValueError):
timeseries(events, "hour", "tokens", 3, now=now)
with pytest.raises(ValueError):
timeseries(events, "minute", "cost", 3, now=now)
async def test_stats_endpoints_shapes(client):
ok = await client.get("/admin/stats/overview?minutes=30", headers=AUTH)
assert ok.status_code == 200
assert {"requests", "tok_in", "cache_hit_rate", "tps_p95", "ttft_p95_ms"} \
<= set(ok.json())
ok2 = await client.get("/admin/stats/timeseries",
params={"bucket": "minute", "metric": "tokens",
"minutes": 15},
headers=AUTH)
body = ok2.json()
assert len(body["buckets"]) == 15
bad = await client.get("/admin/stats/timeseries",
params={"metric": "nope"}, headers=AUTH)
assert bad.status_code == 400
# -- proxy logs & alert rules ---------------------------------------------------------
async def test_proxy_logs_endpoint_empty(client):
r = await client.get("/admin/logs/proxy", headers=AUTH)
assert r.status_code == 200
assert r.json() == {"logs": []}
async def test_proxy_logs_ring_returns_newest_first(client):
"""Ring-backed path: proxy logs must be newest-first and include the
Cache-Control: no-store header. Pins the contract that the frontend
relies on (it no longer reverses the array)."""
app = app_mod.create_app(_config())
async with LifespanManager(app):
# Publish three proxy events in order a, b, c (c is newest).
app.state.wiwi.logs.log_proxy("info", "alpha", request_id="ra")
app.state.wiwi.logs.log_proxy("info", "beta", request_id="rb")
app.state.wiwi.logs.log_proxy("error", "gamma", request_id="rc")
# The ring is pumped asynchronously; give the worker a beat.
import asyncio
await asyncio.sleep(0.05)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport,
base_url="http://test") as c:
r = await c.get("/admin/logs/proxy", headers=AUTH)
assert r.status_code == 200
assert r.headers.get("cache-control") == "no-store"
logs = r.json()["logs"]
# newest-first: gamma (last published) comes first
assert [row["message"] for row in logs] == ["gamma", "beta", "alpha"]
assert [row["level"] for row in logs] == ["error", "info", "info"]
async def test_alert_rules_roundtrip(client):
rules = [{"id": "r1", "webhook_url": "https://hooks.example/x",
"metric": "spend", "threshold": 10.0}]
r = await client.put("/admin/alert-rules", json={"rules": rules}, headers=AUTH)
assert r.status_code == 200
got = await client.get("/admin/alert-rules", headers=AUTH)
assert got.json()["rules"] == rules
bad = await client.put("/admin/alert-rules", json={"rules": "nope"}, headers=AUTH)
assert bad.status_code == 400
# -- SPA mount ------------------------------------------------------------------------
async def test_spa_mount_serves_index_when_built(tmp_path, monkeypatch):
static_dir = tmp_path / "static"
assets = static_dir / "assets"
assets.mkdir(parents=True)
(static_dir / "index.html").write_text("<html><body>wiwi-ui-marker</body></html>")
(assets / "app.js").write_text("console.log('x')")
monkeypatch.setenv("WIWI_STATIC_DIR", str(static_dir))
app = app_mod.create_app(_config())
async with LifespanManager(app):
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport,
base_url="http://test") as c:
r = await c.get("/admin/ui/")
assert r.status_code == 200
assert "text/html" in r.headers["content-type"]
assert "wiwi-ui-marker" in r.text
ra = await c.get("/admin/ui/assets/app.js")
assert ra.status_code == 200
# admin API routes still win over the mount
api = await c.get("/admin/keys", headers=AUTH)
assert api.status_code == 200
def test_default_static_dir_is_package_static(monkeypatch):
monkeypatch.delenv("WIWI_STATIC_DIR", raising=False)
expected = Path(app_mod.__file__).parent / "static"
assert os.path.isdir(expected) or True # dir may not exist until first build
# sanity: env override path construction matches create_app's logic
assert str(expected).endswith("server/static")