"""Test 04 — End-to-end: real server + real SDK fake-trainer subprocess. Boots the FastAPI app with TestClient, starts a REAL python process running the REAL Integration SDK, and verifies the full loop: detect -> connect -> telemetry -> bottleneck -> proposal -> boundary apply -> rollback -> pause -> graceful stop. """ from __future__ import annotations import json import sys import time import pytest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from fastapi.testclient import TestClient from server.app import create_app from tests.conftest import run_fake_trainer, wait_for @pytest.fixture() def config_auto(config): """E2E needs the service actually starting with monitoring active.""" cfg = dict(config) cfg["accelerator"] = {**cfg["accelerator"], "autostart": True, "auto_connect": False} cfg["monitoring"] = {**cfg.get("monitoring", {}), "enabled": False, "autostart": True} return cfg def _wait_exit(proc, timeout=15): try: out, err = proc.communicate(timeout=timeout) except Exception: # noqa: BLE001 proc.kill() out, err = proc.communicate() return out, err def test_full_control_loop(isolated_lpu_home, config_auto): app = create_app(config_auto) with TestClient(app) as client: # 1) status: server up, accelerator started by app startup event st = client.get("/api/status").json() assert st["accelerator_status"] == "running" assert st["runtime_version"].startswith("7."), st["runtime_version"] # 2) hardware snapshot exists (CPU/RAM real; GPU None-honest here) hw = client.get("/api/hardware", params={"force": "1"}).json() assert hw["ram"]["total_gb"] > 0 # 3) launch the fake trainer with the real SDK proc = run_fake_trainer(isolated_lpu_home, iterations=600) try: # 4) session descriptor appears sessions_root = isolated_lpu_home / "sessions" deadline = time.time() + 10 sid = None while time.time() < deadline: if sessions_root.is_dir(): for d in sessions_root.iterdir(): if (d / "session.json").exists(): desc = json.loads((d / "session.json").read_text(encoding="utf-8")) if desc.get("sdk"): sid = desc["session_id"] break if sid: break time.sleep(0.1) assert sid, "SDK session descriptor must appear" desc = json.loads((sessions_root / sid / "session.json") .read_text(encoding="utf-8")) assert desc["sdk"] is True and desc["pid"] == proc.pid # 5) connect via session id res = client.post("/api/connect", json={"session_id": sid}).json() assert res["ok"] is True st = client.get("/api/status").json() assert st["connected"] is True assert st["training"]["attached"] is True assert st["training"]["session_id"] == sid # 6) telemetry flows: wait for metrics + status with phase means deadline = time.time() + 15 phases = None while time.time() < deadline: st = client.get("/api/status").json() phases = (st.get("training") or {}).get("phases_mean_s") if phases: break time.sleep(0.3) assert phases, "phase timings must reach the service from the SDK" assert "data" in phases and "forward" in phases st = client.get("/api/status").json() assert st["training"]["data_wait_ratio"] > 0.5, \ "fake trainer's slow data phase must surface as data-wait" # 7) terminal shows real events ev = client.get("/api/terminal").json()["events"] assert any(e["kind"] == "project" for e in ev) # 8) bottleneck must classify (slow data + data-wait >= 0.5) deadline = time.time() + 20 bn = None while time.time() < deadline: bn = client.get("/api/bottleneck").json()["current"] if bn and "DataLoader" in (bn or {}).get("category", ""): break time.sleep(0.3) assert bn, "bottleneck engine must produce a report" assert "DataLoader" in bn["category"] assert bn["confidence"] in ("Measured",) # 9) optimizer produced data-path proposals from the bottleneck deadline = time.time() + 20 props = [] while time.time() < deadline: props = client.get("/api/proposals").json()["proposals"] if props: break time.sleep(0.3) assert props, "proposals must be derived from measured evidence" prop = next(p for p in props if p["param"] == "dataloader.num_workers") assert prop["level"] == 1 assert prop["evidence"]["data_wait_ratio"] >= 0.5 # 10) apply the proposal (level 1, auto-apply enabled in config) res = client.post(f"/api/proposals/{prop['id']}/apply", json={"confirm": True}).json() assert res["status"] in ("applied", "queued_at_boundary") # the fake trainer only applies at boundary() -> queued expected assert res["status"] == "queued_at_boundary" # 11) trainer crosses the boundary at its next pause_point() deadline = time.time() + 15 ack_path = sessions_root / sid / "acks.jsonl" applied_ok = False while time.time() < deadline: if ack_path.exists(): for line in ack_path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: a = json.loads(line) except json.JSONDecodeError: continue if (a.get("cmd") == "set_parameter" and a.get("param") == "dataloader.num_workers" and a.get("status") == "applied"): applied_ok = True break if applied_ok: break time.sleep(0.2) assert applied_ok, "SDK must ack the boundary application" # 12) ledger records the change changes = client.get("/api/changes").json()["changes"] assert any(c["param"] == "dataloader.num_workers" for c in changes) # 13) status settings reflect the LIVE value deadline = time.time() + 10 live = None while time.time() < deadline: st = client.get("/api/status").json() settings = ((st.get("training") or {}).get("settings") or {}) live = settings.get("dataloader.num_workers", settings.get("num_workers")) if live == 4: break time.sleep(0.3) assert live == 4, "live settings must show the applied value" # 14) pause / resume through the API res = client.post("/api/training/pause").json() assert res["ok"] is True deadline = time.time() + 8 while time.time() < deadline: st = client.get("/api/status").json() if (st.get("training") or {}).get("state") == "paused": break time.sleep(0.2) assert (client.get("/api/status").json() .get("training", {}).get("state")) == "paused" res = client.post("/api/training/resume").json() assert res["ok"] is True # 15) rollback the change changes = client.get("/api/changes").json()["changes"] ch = next(c for c in changes if c["param"] == "dataloader.num_workers" and c["status"] in ("applied", "queued_at_boundary")) res = client.post(f"/api/changes/{ch['id']}/rollback").json() assert res["ok"] is True # 16) graceful stop: trainer exits by itself at its boundary res = client.post("/api/training/stop", json={"mode": "graceful"}).json() assert res["ok"] is True out, err = _wait_exit(proc, timeout=20) # trainer finished cleanly (finalize writes status finished) st_file = sessions_root / sid / "status.json" final = json.loads(wait_for(st_file, timeout=10)) assert final["state"] == "finished" assert final.get("final_psnr") == 31.62 assert proc.returncode in (0, None) finally: if proc.poll() is None: proc.kill() _wait_exit(proc) def test_websocket_stream(isolated_lpu_home, config_auto): import subprocess script = isolated_lpu_home / "fake_train_edsrc.py" script.write_text("import time\ntime.sleep(30)\n", encoding="utf-8") proc = subprocess.Popen([sys.executable, str(script)]) try: app = create_app(config_auto) with TestClient(app) as client: with client.websocket_connect("/ws") as ws: # snapshot drains first; a discovered training-like process # must then stream live project events deadline = time.time() + 10 got = None client.post("/api/detect") while time.time() < deadline: msg = json.loads(ws.receive_text()) if msg.get("kind") == "project": got = msg break assert got is not None, "discovery events must stream over WebSocket" # disconnect mid-stream must not break the bus for later subscribers with client.websocket_connect("/ws") as ws2: client.post("/api/monitoring/stop") msg = json.loads(ws2.receive_text()) assert msg["kind"] in ("lpu", "heartbeat", "project") finally: if proc.poll() is None: proc.kill() proc.wait(timeout=5) def test_accelerator_stop_leaves_training_alone(isolated_lpu_home, config_auto): """Mission rule: STOP ACCELERATOR never stops training.""" app = create_app(config_auto) with TestClient(app) as client: proc = run_fake_trainer(isolated_lpu_home, iterations=400) try: time.sleep(1.0) assert proc.poll() is None, "trainer must be running" res = client.post("/api/accelerator/stop").json() assert res["ok"] is True time.sleep(1.5) assert proc.poll() is None, \ "trainer must STILL be running after accelerator stop" # and the server itself is still up st = client.get("/api/status").json() assert st["accelerator_status"] == "stopped" st2 = client.post("/api/accelerator/start").json() assert st2["ok"] is True finally: if proc.poll() is None: client.post("/api/training/stop", json={"mode": "emergency", "consent": True}) _wait_exit(proc, timeout=10) if proc.poll() is None: proc.kill() _wait_exit(proc) def test_emergency_stop_requires_consent(isolated_lpu_home, config_auto): app = create_app(config_auto) with TestClient(app) as client: proc = run_fake_trainer(isolated_lpu_home, iterations=400) try: # attach to the trainer's SDK session first (emergency stop acts # on the ATTACHED process) deadline = time.time() + 10 sid = None root = isolated_lpu_home / "sessions" while time.time() < deadline: if root.is_dir(): for d in root.iterdir(): if (d / "session.json").exists(): sid = d.name break if sid: break time.sleep(0.1) assert sid assert client.post("/api/connect", json={"session_id": sid}).json()["ok"] time.sleep(0.3) # without consent: 400 r = client.post("/api/training/stop", json={"mode": "emergency", "consent": False}) assert r.status_code == 400 assert proc.poll() is None, "no consent -> process untouched" # with consent: terminated r = client.post("/api/training/stop", json={"mode": "emergency", "consent": True}) assert r.status_code == 200 _wait_exit(proc, timeout=10) assert proc.poll() is not None finally: if proc.poll() is None: proc.kill() _wait_exit(proc) def test_report_export_formats(isolated_lpu_home, config_auto): app = create_app(config_auto) with TestClient(app) as client: for fmt in ("json", "csv", "html"): r = client.get("/api/report/export", params={"format": fmt}) assert r.status_code == 200, f"{fmt} export failed" path = r.json()["path"] assert Path(path).exists() and Path(path).stat().st_size > 0 # honesty note present in the JSON report rep = json.loads(Path(client.get("/api/report/export", params={"format": "json"}).json()["path"]) .read_text(encoding="utf-8")) assert "NEVER filled with estimates" in rep["honesty_note"] def test_dashboard_served(isolated_lpu_home, config_auto): app = create_app(config_auto) with TestClient(app) as client: r = client.get("/") assert r.status_code == 200 assert "LPU AI ACCELERATOR" in r.text assert "START ACCELERATOR" in r.text assert "EMERGENCY STOP" in r.text