Spaces:
Sleeping
Sleeping
File size: 11,104 Bytes
3deca49 75878d6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 | """
tests/test_api.py
================
ECOSYSTEM pytest suite for the FastAPI surface (contract section 2).
Covers (contract section 5 / test descriptions):
* GET /data/stats -> 200
* GET /model/info -> returns config
* POST /data/add -> 200 with added count (private-training mode + key)
* POST /generate -> 200 (untrained model is fine) returns
text + tokens_generated
* GET /train/status -> 200 with status fields
* Deploy-mode gating: /data/add and /train/start return 403 in demo mode
and when the X-API-Key is missing in private mode.
The API DB path is steered to a temp file via the TMT_DB_PATH env var so the
real metatron.db is never touched (contract rule 0.6). Deploy mode + API key
are read at request time by api.py, so the fixtures set them via monkeypatch.
"""
from __future__ import annotations
import os
import sys
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if _REPO_ROOT not in sys.path:
sys.path.insert(0, _REPO_ROOT)
import pytest
# Import the app after the env var is set in the fixture below.
from fastapi.testclient import TestClient
# Texts long enough to clear the quality threshold (>=0.4).
_ADD_TEXTS = [
"The firewall enforces a zero-trust policy with mfa and tls encryption.",
"Sparse attention masks reduce the quadratic cost of self-attention.",
"Mixture of experts routes tokens to specialized feed forward networks.",
"The transformer model uses attention, embeddings and a softmax over logits.",
]
_ADMIN_HEADERS = {"X-API-Key": "testkey"}
@pytest.fixture
def client(tmp_path, monkeypatch):
"""Private-training mode client: admin endpoints enabled with a test key."""
db_path = str(tmp_path / "api_test.db")
monkeypatch.setenv("TMT_DB_PATH", db_path)
monkeypatch.setenv("TMT_DEPLOY_MODE", "private-training")
monkeypatch.setenv("TMT_API_KEY", "testkey")
import api as api_mod
with TestClient(api_mod.app) as c:
yield c
@pytest.fixture
def demo_client(tmp_path, monkeypatch):
"""Demo mode client: admin endpoints disabled regardless of key."""
db_path = str(tmp_path / "api_demo.db")
monkeypatch.setenv("TMT_DB_PATH", db_path)
monkeypatch.setenv("TMT_DEPLOY_MODE", "demo")
monkeypatch.setenv("TMT_API_KEY", "testkey")
import api as api_mod
with TestClient(api_mod.app) as c:
yield c
# ββ GET /health -> 200 ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def test_health_ok(client: TestClient) -> None:
r = client.get("/health")
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
assert r.json() == {"status": "ok"}
# ββ GET /data/stats -> 200, contract keys present βββββββββββββββββββββββββββββ
def test_data_stats_ok(client: TestClient) -> None:
r = client.get("/data/stats")
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
body = r.json()
for k in ("total", "by_domain", "avg_quality", "used_in_training"):
assert k in body, f"/data/stats missing key {k}"
# ββ GET /model/info -> returns config βββββββββββββββββββββββββββββββββββββββββ
def test_model_info_returns_config(client: TestClient) -> None:
r = client.get("/model/info")
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
info = r.json()
assert "config" in info and isinstance(info["config"], dict)
from config import CONFIG
assert info["config"].get("vocab_size") == CONFIG["vocab_size"]
assert "active_checkpoint" in info
# ββ POST /data/add -> 200 with added count (private mode + key) ββββββββββββββββ
def test_data_add_returns_added_count(client: TestClient) -> None:
r = client.post("/data/add", json={
"texts": _ADD_TEXTS, "domain": "cybersecurity",
"quality_threshold": 0.4,
}, headers=_ADMIN_HEADERS)
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
body = r.json()
assert body["added"] >= 1, f"added>=1, got {body['added']}"
assert body["added"] + body["rejected"] == len(_ADD_TEXTS)
assert body["domain"] == "cybersecurity"
# ββ POST /generate -> 200, returns text + tokens_generated (untrained ok) βββββ
def test_generate_returns_text_and_tokens(client: TestClient) -> None:
r = client.post("/generate", json={
"prompt": "firewall ", "max_length": 8, "temperature": 0.7,
})
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
gen = r.json()
assert "text" in gen and isinstance(gen["text"], str)
assert "tokens_generated" in gen
assert isinstance(gen["tokens_generated"], int)
assert gen["tokens_generated"] >= 0
# step==0 for an untrained model (no checkpoint in a fresh temp DB)
assert gen.get("step") == 0
# ββ /generate input bounds: prompt over 2000 chars -> 422 βββββββββββββββββββββ
def test_generate_rejects_oversized_prompt(client: TestClient) -> None:
r = client.post("/generate", json={
"prompt": "x" * 2001, "max_length": 4, "temperature": 0.7,
})
assert r.status_code == 422, f"expected 422, got {r.status_code}: {r.text}"
# ββ /generate output bound: max_length over 256 -> 422 ββββββββββββββββββββββββ
def test_generate_rejects_oversized_max_length(client: TestClient) -> None:
r = client.post("/generate", json={
"prompt": "ok", "max_length": 257, "temperature": 0.7,
})
assert r.status_code == 422, f"expected 422, got {r.status_code}: {r.text}"
# ββ GET /train/status -> 200 with status fields βββββββββββββββββββββββββββββββ
def test_train_status_ok(client: TestClient) -> None:
r = client.get("/train/status")
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
st = r.json()
for k in ("is_training", "current_step", "total_steps", "current_loss"):
assert k in st, f"/train/status missing {k}"
assert st["is_training"] is False
# ββ Admin gate: private mode but no key -> 403 ββββββββββββββββββββββββββββββββ
def test_data_add_requires_key(client: TestClient) -> None:
r = client.post("/data/add", json={
"texts": _ADD_TEXTS, "domain": "cybersecurity",
"quality_threshold": 0.4,
}) # no X-API-Key header
assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}"
def test_data_add_wrong_key(client: TestClient) -> None:
r = client.post("/data/add", json={
"texts": _ADD_TEXTS, "domain": "cybersecurity",
"quality_threshold": 0.4,
}, headers={"X-API-Key": "wrong"})
assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}"
# ββ Demo mode: admin endpoints disabled even with a valid key -> 403 ββββββββββ
def test_data_add_forbidden_in_demo(demo_client: TestClient) -> None:
r = demo_client.post("/data/add", json={
"texts": _ADD_TEXTS, "domain": "cybersecurity",
"quality_threshold": 0.4,
}, headers=_ADMIN_HEADERS)
assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}"
def test_train_start_forbidden_in_demo(demo_client: TestClient) -> None:
r = demo_client.post("/train/start", json={
"steps": 1, "domain": "general", "min_quality": 0.4,
}, headers=_ADMIN_HEADERS)
assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}"
# ββ Demo mode: read-only endpoints still public ββββββββββββββββββββββββββββββ
def test_readonly_endpoints_ok_in_demo(demo_client: TestClient) -> None:
assert demo_client.get("/health").status_code == 200
assert demo_client.get("/data/stats").status_code == 200
assert demo_client.get("/model/info").status_code == 200
assert demo_client.get("/train/status").status_code == 200
assert demo_client.post("/generate", json={
"prompt": "hi", "max_length": 4, "temperature": 0.7,
}).status_code == 200
# ββ POST /ask -> contract shape {decision, route, evidence, answer} βββββββββββ
@pytest.fixture
def ask_client(tmp_path, monkeypatch):
"""Private-training client with a minimal quantum corpus DB for /ask."""
db_path = str(tmp_path / "api_test.db")
qdb_path = str(tmp_path / "quantum_corpus.db")
monkeypatch.setenv("TMT_DB_PATH", db_path)
monkeypatch.setenv("TMT_DEPLOY_MODE", "private-training")
monkeypatch.setenv("TMT_API_KEY", "testkey")
monkeypatch.setenv("TMT_QUANTUM_CORPUS_DB", qdb_path)
# Build a tiny corpus with train+val records so _get_ask_retriever succeeds.
from quantum_corpus import schema as _qschema
_qschema.init_db(qdb_path)
_qschema.write_records(qdb_path, [
_qschema.Record(
source_type="manifest", project="wormhole", doc_id="w:1",
text="Circuit 1: OTOC Lyapunov Exponent Measurement. "
"Measures the out-of-time-ordered correlator on ibm_kingston.",
split="train", sensitivity="public",
),
_qschema.Record(
source_type="ibm_job", project="ibm-quantum", doc_id="ibm:d5a6",
text="IBM Quantum job d5a6 on backend ibm_fez, status Completed, "
"program sampler, 8192 samples.",
split="train", sensitivity="internal",
),
_qschema.Record(
source_type="repo", project="GRE", doc_id="GRE:readme",
text="Sierpinski triangle quantum walk with golden silver bronze coin angles.",
split="val", sensitivity="public",
),
])
import api as api_mod
# Clear any cached retriever from a previous import in this process.
api_mod._ask_retriever_cache.update(retriever=None, db_path=None, mtime=None)
with TestClient(api_mod.app) as c:
yield c
def test_ask_contract_shape(ask_client: TestClient) -> None:
r = ask_client.post("/ask", json={
"question": "what does the OTOC circuit measure?",
"top_k": 3,
"mode": "quantum-private",
"sensitivity": "internal",
}, headers=_ADMIN_HEADERS)
assert r.status_code == 200, f"status {r.status_code}: {r.text}"
body = r.json()
for k in ("decision", "route", "evidence", "answer"):
assert k in body, f"/ask response missing contract key {k}"
assert body["route"] in ("retrieval", "structured", "declined")
assert isinstance(body["evidence"], list)
assert isinstance(body["answer"], str)
# The user-facing answer must NOT come from the 32-token sampler; the
# contract response does not include a "generated" field produced by it.
assert "generated" not in body
|