Virtual-LPU / tests /test_04_e2e_api.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
14.6 kB
"""Test 04 — End-to-end: real server + real SDK fake-trainer subprocess.
Boots the FastAPI app with TestClient, starts a REAL python process running
the REAL Integration SDK, and verifies the full loop:
detect -> connect -> telemetry -> bottleneck -> proposal -> boundary apply
-> rollback -> pause -> graceful stop.
"""
from __future__ import annotations
import json
import sys
import time
import pytest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from fastapi.testclient import TestClient
from server.app import create_app
from tests.conftest import run_fake_trainer, wait_for
@pytest.fixture()
def config_auto(config):
"""E2E needs the service actually starting with monitoring active."""
cfg = dict(config)
cfg["accelerator"] = {**cfg["accelerator"], "autostart": True,
"auto_connect": False}
cfg["monitoring"] = {**cfg.get("monitoring", {}),
"enabled": False, "autostart": True}
return cfg
def _wait_exit(proc, timeout=15):
try:
out, err = proc.communicate(timeout=timeout)
except Exception: # noqa: BLE001
proc.kill()
out, err = proc.communicate()
return out, err
def test_full_control_loop(isolated_lpu_home, config_auto):
app = create_app(config_auto)
with TestClient(app) as client:
# 1) status: server up, accelerator started by app startup event
st = client.get("/api/status").json()
assert st["accelerator_status"] == "running"
assert st["runtime_version"].startswith("7."), st["runtime_version"]
# 2) hardware snapshot exists (CPU/RAM real; GPU None-honest here)
hw = client.get("/api/hardware", params={"force": "1"}).json()
assert hw["ram"]["total_gb"] > 0
# 3) launch the fake trainer with the real SDK
proc = run_fake_trainer(isolated_lpu_home, iterations=600)
try:
# 4) session descriptor appears
sessions_root = isolated_lpu_home / "sessions"
deadline = time.time() + 10
sid = None
while time.time() < deadline:
if sessions_root.is_dir():
for d in sessions_root.iterdir():
if (d / "session.json").exists():
desc = json.loads((d / "session.json").read_text(encoding="utf-8"))
if desc.get("sdk"):
sid = desc["session_id"]
break
if sid:
break
time.sleep(0.1)
assert sid, "SDK session descriptor must appear"
desc = json.loads((sessions_root / sid / "session.json")
.read_text(encoding="utf-8"))
assert desc["sdk"] is True and desc["pid"] == proc.pid
# 5) connect via session id
res = client.post("/api/connect", json={"session_id": sid}).json()
assert res["ok"] is True
st = client.get("/api/status").json()
assert st["connected"] is True
assert st["training"]["attached"] is True
assert st["training"]["session_id"] == sid
# 6) telemetry flows: wait for metrics + status with phase means
deadline = time.time() + 15
phases = None
while time.time() < deadline:
st = client.get("/api/status").json()
phases = (st.get("training") or {}).get("phases_mean_s")
if phases:
break
time.sleep(0.3)
assert phases, "phase timings must reach the service from the SDK"
assert "data" in phases and "forward" in phases
st = client.get("/api/status").json()
assert st["training"]["data_wait_ratio"] > 0.5, \
"fake trainer's slow data phase must surface as data-wait"
# 7) terminal shows real events
ev = client.get("/api/terminal").json()["events"]
assert any(e["kind"] == "project" for e in ev)
# 8) bottleneck must classify (slow data + data-wait >= 0.5)
deadline = time.time() + 20
bn = None
while time.time() < deadline:
bn = client.get("/api/bottleneck").json()["current"]
if bn and "DataLoader" in (bn or {}).get("category", ""):
break
time.sleep(0.3)
assert bn, "bottleneck engine must produce a report"
assert "DataLoader" in bn["category"]
assert bn["confidence"] in ("Measured",)
# 9) optimizer produced data-path proposals from the bottleneck
deadline = time.time() + 20
props = []
while time.time() < deadline:
props = client.get("/api/proposals").json()["proposals"]
if props:
break
time.sleep(0.3)
assert props, "proposals must be derived from measured evidence"
prop = next(p for p in props
if p["param"] == "dataloader.num_workers")
assert prop["level"] == 1
assert prop["evidence"]["data_wait_ratio"] >= 0.5
# 10) apply the proposal (level 1, auto-apply enabled in config)
res = client.post(f"/api/proposals/{prop['id']}/apply",
json={"confirm": True}).json()
assert res["status"] in ("applied", "queued_at_boundary")
# the fake trainer only applies at boundary() -> queued expected
assert res["status"] == "queued_at_boundary"
# 11) trainer crosses the boundary at its next pause_point()
deadline = time.time() + 15
ack_path = sessions_root / sid / "acks.jsonl"
applied_ok = False
while time.time() < deadline:
if ack_path.exists():
for line in ack_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
a = json.loads(line)
except json.JSONDecodeError:
continue
if (a.get("cmd") == "set_parameter"
and a.get("param") == "dataloader.num_workers"
and a.get("status") == "applied"):
applied_ok = True
break
if applied_ok:
break
time.sleep(0.2)
assert applied_ok, "SDK must ack the boundary application"
# 12) ledger records the change
changes = client.get("/api/changes").json()["changes"]
assert any(c["param"] == "dataloader.num_workers" for c in changes)
# 13) status settings reflect the LIVE value
deadline = time.time() + 10
live = None
while time.time() < deadline:
st = client.get("/api/status").json()
settings = ((st.get("training") or {}).get("settings") or {})
live = settings.get("dataloader.num_workers", settings.get("num_workers"))
if live == 4:
break
time.sleep(0.3)
assert live == 4, "live settings must show the applied value"
# 14) pause / resume through the API
res = client.post("/api/training/pause").json()
assert res["ok"] is True
deadline = time.time() + 8
while time.time() < deadline:
st = client.get("/api/status").json()
if (st.get("training") or {}).get("state") == "paused":
break
time.sleep(0.2)
assert (client.get("/api/status").json()
.get("training", {}).get("state")) == "paused"
res = client.post("/api/training/resume").json()
assert res["ok"] is True
# 15) rollback the change
changes = client.get("/api/changes").json()["changes"]
ch = next(c for c in changes
if c["param"] == "dataloader.num_workers"
and c["status"] in ("applied", "queued_at_boundary"))
res = client.post(f"/api/changes/{ch['id']}/rollback").json()
assert res["ok"] is True
# 16) graceful stop: trainer exits by itself at its boundary
res = client.post("/api/training/stop",
json={"mode": "graceful"}).json()
assert res["ok"] is True
out, err = _wait_exit(proc, timeout=20)
# trainer finished cleanly (finalize writes status finished)
st_file = sessions_root / sid / "status.json"
final = json.loads(wait_for(st_file, timeout=10))
assert final["state"] == "finished"
assert final.get("final_psnr") == 31.62
assert proc.returncode in (0, None)
finally:
if proc.poll() is None:
proc.kill()
_wait_exit(proc)
def test_websocket_stream(isolated_lpu_home, config_auto):
import subprocess
script = isolated_lpu_home / "fake_train_edsrc.py"
script.write_text("import time\ntime.sleep(30)\n", encoding="utf-8")
proc = subprocess.Popen([sys.executable, str(script)])
try:
app = create_app(config_auto)
with TestClient(app) as client:
with client.websocket_connect("/ws") as ws:
# snapshot drains first; a discovered training-like process
# must then stream live project events
deadline = time.time() + 10
got = None
client.post("/api/detect")
while time.time() < deadline:
msg = json.loads(ws.receive_text())
if msg.get("kind") == "project":
got = msg
break
assert got is not None, "discovery events must stream over WebSocket"
# disconnect mid-stream must not break the bus for later subscribers
with client.websocket_connect("/ws") as ws2:
client.post("/api/monitoring/stop")
msg = json.loads(ws2.receive_text())
assert msg["kind"] in ("lpu", "heartbeat", "project")
finally:
if proc.poll() is None:
proc.kill()
proc.wait(timeout=5)
def test_accelerator_stop_leaves_training_alone(isolated_lpu_home, config_auto):
"""Mission rule: STOP ACCELERATOR never stops training."""
app = create_app(config_auto)
with TestClient(app) as client:
proc = run_fake_trainer(isolated_lpu_home, iterations=400)
try:
time.sleep(1.0)
assert proc.poll() is None, "trainer must be running"
res = client.post("/api/accelerator/stop").json()
assert res["ok"] is True
time.sleep(1.5)
assert proc.poll() is None, \
"trainer must STILL be running after accelerator stop"
# and the server itself is still up
st = client.get("/api/status").json()
assert st["accelerator_status"] == "stopped"
st2 = client.post("/api/accelerator/start").json()
assert st2["ok"] is True
finally:
if proc.poll() is None:
client.post("/api/training/stop", json={"mode": "emergency",
"consent": True})
_wait_exit(proc, timeout=10)
if proc.poll() is None:
proc.kill()
_wait_exit(proc)
def test_emergency_stop_requires_consent(isolated_lpu_home, config_auto):
app = create_app(config_auto)
with TestClient(app) as client:
proc = run_fake_trainer(isolated_lpu_home, iterations=400)
try:
# attach to the trainer's SDK session first (emergency stop acts
# on the ATTACHED process)
deadline = time.time() + 10
sid = None
root = isolated_lpu_home / "sessions"
while time.time() < deadline:
if root.is_dir():
for d in root.iterdir():
if (d / "session.json").exists():
sid = d.name
break
if sid:
break
time.sleep(0.1)
assert sid
assert client.post("/api/connect", json={"session_id": sid}).json()["ok"]
time.sleep(0.3)
# without consent: 400
r = client.post("/api/training/stop",
json={"mode": "emergency", "consent": False})
assert r.status_code == 400
assert proc.poll() is None, "no consent -> process untouched"
# with consent: terminated
r = client.post("/api/training/stop",
json={"mode": "emergency", "consent": True})
assert r.status_code == 200
_wait_exit(proc, timeout=10)
assert proc.poll() is not None
finally:
if proc.poll() is None:
proc.kill()
_wait_exit(proc)
def test_report_export_formats(isolated_lpu_home, config_auto):
app = create_app(config_auto)
with TestClient(app) as client:
for fmt in ("json", "csv", "html"):
r = client.get("/api/report/export", params={"format": fmt})
assert r.status_code == 200, f"{fmt} export failed"
path = r.json()["path"]
assert Path(path).exists() and Path(path).stat().st_size > 0
# honesty note present in the JSON report
rep = json.loads(Path(client.get("/api/report/export",
params={"format": "json"}).json()["path"])
.read_text(encoding="utf-8"))
assert "NEVER filled with estimates" in rep["honesty_note"]
def test_dashboard_served(isolated_lpu_home, config_auto):
app = create_app(config_auto)
with TestClient(app) as client:
r = client.get("/")
assert r.status_code == 200
assert "LPU AI ACCELERATOR" in r.text
assert "START ACCELERATOR" in r.text
assert "EMERGENCY STOP" in r.text