Spaces:
Running
Running
Download tests/test_04_e2e_api.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 14.6 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/tests/test_04_e2e_api.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/tests/test_04_e2e_api.py
-
curl -L -o test_04_e2e_api.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/tests/test_04_e2e_api.py
14.6 kB
| """Test 04 — End-to-end: real server + real SDK fake-trainer subprocess. | |
| Boots the FastAPI app with TestClient, starts a REAL python process running | |
| the REAL Integration SDK, and verifies the full loop: | |
| detect -> connect -> telemetry -> bottleneck -> proposal -> boundary apply | |
| -> rollback -> pause -> graceful stop. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| import time | |
| import pytest | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from fastapi.testclient import TestClient | |
| from server.app import create_app | |
| from tests.conftest import run_fake_trainer, wait_for | |
| def config_auto(config): | |
| """E2E needs the service actually starting with monitoring active.""" | |
| cfg = dict(config) | |
| cfg["accelerator"] = {**cfg["accelerator"], "autostart": True, | |
| "auto_connect": False} | |
| cfg["monitoring"] = {**cfg.get("monitoring", {}), | |
| "enabled": False, "autostart": True} | |
| return cfg | |
| def _wait_exit(proc, timeout=15): | |
| try: | |
| out, err = proc.communicate(timeout=timeout) | |
| except Exception: # noqa: BLE001 | |
| proc.kill() | |
| out, err = proc.communicate() | |
| return out, err | |
| def test_full_control_loop(isolated_lpu_home, config_auto): | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| # 1) status: server up, accelerator started by app startup event | |
| st = client.get("/api/status").json() | |
| assert st["accelerator_status"] == "running" | |
| assert st["runtime_version"].startswith("7."), st["runtime_version"] | |
| # 2) hardware snapshot exists (CPU/RAM real; GPU None-honest here) | |
| hw = client.get("/api/hardware", params={"force": "1"}).json() | |
| assert hw["ram"]["total_gb"] > 0 | |
| # 3) launch the fake trainer with the real SDK | |
| proc = run_fake_trainer(isolated_lpu_home, iterations=600) | |
| try: | |
| # 4) session descriptor appears | |
| sessions_root = isolated_lpu_home / "sessions" | |
| deadline = time.time() + 10 | |
| sid = None | |
| while time.time() < deadline: | |
| if sessions_root.is_dir(): | |
| for d in sessions_root.iterdir(): | |
| if (d / "session.json").exists(): | |
| desc = json.loads((d / "session.json").read_text(encoding="utf-8")) | |
| if desc.get("sdk"): | |
| sid = desc["session_id"] | |
| break | |
| if sid: | |
| break | |
| time.sleep(0.1) | |
| assert sid, "SDK session descriptor must appear" | |
| desc = json.loads((sessions_root / sid / "session.json") | |
| .read_text(encoding="utf-8")) | |
| assert desc["sdk"] is True and desc["pid"] == proc.pid | |
| # 5) connect via session id | |
| res = client.post("/api/connect", json={"session_id": sid}).json() | |
| assert res["ok"] is True | |
| st = client.get("/api/status").json() | |
| assert st["connected"] is True | |
| assert st["training"]["attached"] is True | |
| assert st["training"]["session_id"] == sid | |
| # 6) telemetry flows: wait for metrics + status with phase means | |
| deadline = time.time() + 15 | |
| phases = None | |
| while time.time() < deadline: | |
| st = client.get("/api/status").json() | |
| phases = (st.get("training") or {}).get("phases_mean_s") | |
| if phases: | |
| break | |
| time.sleep(0.3) | |
| assert phases, "phase timings must reach the service from the SDK" | |
| assert "data" in phases and "forward" in phases | |
| st = client.get("/api/status").json() | |
| assert st["training"]["data_wait_ratio"] > 0.5, \ | |
| "fake trainer's slow data phase must surface as data-wait" | |
| # 7) terminal shows real events | |
| ev = client.get("/api/terminal").json()["events"] | |
| assert any(e["kind"] == "project" for e in ev) | |
| # 8) bottleneck must classify (slow data + data-wait >= 0.5) | |
| deadline = time.time() + 20 | |
| bn = None | |
| while time.time() < deadline: | |
| bn = client.get("/api/bottleneck").json()["current"] | |
| if bn and "DataLoader" in (bn or {}).get("category", ""): | |
| break | |
| time.sleep(0.3) | |
| assert bn, "bottleneck engine must produce a report" | |
| assert "DataLoader" in bn["category"] | |
| assert bn["confidence"] in ("Measured",) | |
| # 9) optimizer produced data-path proposals from the bottleneck | |
| deadline = time.time() + 20 | |
| props = [] | |
| while time.time() < deadline: | |
| props = client.get("/api/proposals").json()["proposals"] | |
| if props: | |
| break | |
| time.sleep(0.3) | |
| assert props, "proposals must be derived from measured evidence" | |
| prop = next(p for p in props | |
| if p["param"] == "dataloader.num_workers") | |
| assert prop["level"] == 1 | |
| assert prop["evidence"]["data_wait_ratio"] >= 0.5 | |
| # 10) apply the proposal (level 1, auto-apply enabled in config) | |
| res = client.post(f"/api/proposals/{prop['id']}/apply", | |
| json={"confirm": True}).json() | |
| assert res["status"] in ("applied", "queued_at_boundary") | |
| # the fake trainer only applies at boundary() -> queued expected | |
| assert res["status"] == "queued_at_boundary" | |
| # 11) trainer crosses the boundary at its next pause_point() | |
| deadline = time.time() + 15 | |
| ack_path = sessions_root / sid / "acks.jsonl" | |
| applied_ok = False | |
| while time.time() < deadline: | |
| if ack_path.exists(): | |
| for line in ack_path.read_text(encoding="utf-8").splitlines(): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| a = json.loads(line) | |
| except json.JSONDecodeError: | |
| continue | |
| if (a.get("cmd") == "set_parameter" | |
| and a.get("param") == "dataloader.num_workers" | |
| and a.get("status") == "applied"): | |
| applied_ok = True | |
| break | |
| if applied_ok: | |
| break | |
| time.sleep(0.2) | |
| assert applied_ok, "SDK must ack the boundary application" | |
| # 12) ledger records the change | |
| changes = client.get("/api/changes").json()["changes"] | |
| assert any(c["param"] == "dataloader.num_workers" for c in changes) | |
| # 13) status settings reflect the LIVE value | |
| deadline = time.time() + 10 | |
| live = None | |
| while time.time() < deadline: | |
| st = client.get("/api/status").json() | |
| settings = ((st.get("training") or {}).get("settings") or {}) | |
| live = settings.get("dataloader.num_workers", settings.get("num_workers")) | |
| if live == 4: | |
| break | |
| time.sleep(0.3) | |
| assert live == 4, "live settings must show the applied value" | |
| # 14) pause / resume through the API | |
| res = client.post("/api/training/pause").json() | |
| assert res["ok"] is True | |
| deadline = time.time() + 8 | |
| while time.time() < deadline: | |
| st = client.get("/api/status").json() | |
| if (st.get("training") or {}).get("state") == "paused": | |
| break | |
| time.sleep(0.2) | |
| assert (client.get("/api/status").json() | |
| .get("training", {}).get("state")) == "paused" | |
| res = client.post("/api/training/resume").json() | |
| assert res["ok"] is True | |
| # 15) rollback the change | |
| changes = client.get("/api/changes").json()["changes"] | |
| ch = next(c for c in changes | |
| if c["param"] == "dataloader.num_workers" | |
| and c["status"] in ("applied", "queued_at_boundary")) | |
| res = client.post(f"/api/changes/{ch['id']}/rollback").json() | |
| assert res["ok"] is True | |
| # 16) graceful stop: trainer exits by itself at its boundary | |
| res = client.post("/api/training/stop", | |
| json={"mode": "graceful"}).json() | |
| assert res["ok"] is True | |
| out, err = _wait_exit(proc, timeout=20) | |
| # trainer finished cleanly (finalize writes status finished) | |
| st_file = sessions_root / sid / "status.json" | |
| final = json.loads(wait_for(st_file, timeout=10)) | |
| assert final["state"] == "finished" | |
| assert final.get("final_psnr") == 31.62 | |
| assert proc.returncode in (0, None) | |
| finally: | |
| if proc.poll() is None: | |
| proc.kill() | |
| _wait_exit(proc) | |
| def test_websocket_stream(isolated_lpu_home, config_auto): | |
| import subprocess | |
| script = isolated_lpu_home / "fake_train_edsrc.py" | |
| script.write_text("import time\ntime.sleep(30)\n", encoding="utf-8") | |
| proc = subprocess.Popen([sys.executable, str(script)]) | |
| try: | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| with client.websocket_connect("/ws") as ws: | |
| # snapshot drains first; a discovered training-like process | |
| # must then stream live project events | |
| deadline = time.time() + 10 | |
| got = None | |
| client.post("/api/detect") | |
| while time.time() < deadline: | |
| msg = json.loads(ws.receive_text()) | |
| if msg.get("kind") == "project": | |
| got = msg | |
| break | |
| assert got is not None, "discovery events must stream over WebSocket" | |
| # disconnect mid-stream must not break the bus for later subscribers | |
| with client.websocket_connect("/ws") as ws2: | |
| client.post("/api/monitoring/stop") | |
| msg = json.loads(ws2.receive_text()) | |
| assert msg["kind"] in ("lpu", "heartbeat", "project") | |
| finally: | |
| if proc.poll() is None: | |
| proc.kill() | |
| proc.wait(timeout=5) | |
| def test_accelerator_stop_leaves_training_alone(isolated_lpu_home, config_auto): | |
| """Mission rule: STOP ACCELERATOR never stops training.""" | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| proc = run_fake_trainer(isolated_lpu_home, iterations=400) | |
| try: | |
| time.sleep(1.0) | |
| assert proc.poll() is None, "trainer must be running" | |
| res = client.post("/api/accelerator/stop").json() | |
| assert res["ok"] is True | |
| time.sleep(1.5) | |
| assert proc.poll() is None, \ | |
| "trainer must STILL be running after accelerator stop" | |
| # and the server itself is still up | |
| st = client.get("/api/status").json() | |
| assert st["accelerator_status"] == "stopped" | |
| st2 = client.post("/api/accelerator/start").json() | |
| assert st2["ok"] is True | |
| finally: | |
| if proc.poll() is None: | |
| client.post("/api/training/stop", json={"mode": "emergency", | |
| "consent": True}) | |
| _wait_exit(proc, timeout=10) | |
| if proc.poll() is None: | |
| proc.kill() | |
| _wait_exit(proc) | |
| def test_emergency_stop_requires_consent(isolated_lpu_home, config_auto): | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| proc = run_fake_trainer(isolated_lpu_home, iterations=400) | |
| try: | |
| # attach to the trainer's SDK session first (emergency stop acts | |
| # on the ATTACHED process) | |
| deadline = time.time() + 10 | |
| sid = None | |
| root = isolated_lpu_home / "sessions" | |
| while time.time() < deadline: | |
| if root.is_dir(): | |
| for d in root.iterdir(): | |
| if (d / "session.json").exists(): | |
| sid = d.name | |
| break | |
| if sid: | |
| break | |
| time.sleep(0.1) | |
| assert sid | |
| assert client.post("/api/connect", json={"session_id": sid}).json()["ok"] | |
| time.sleep(0.3) | |
| # without consent: 400 | |
| r = client.post("/api/training/stop", | |
| json={"mode": "emergency", "consent": False}) | |
| assert r.status_code == 400 | |
| assert proc.poll() is None, "no consent -> process untouched" | |
| # with consent: terminated | |
| r = client.post("/api/training/stop", | |
| json={"mode": "emergency", "consent": True}) | |
| assert r.status_code == 200 | |
| _wait_exit(proc, timeout=10) | |
| assert proc.poll() is not None | |
| finally: | |
| if proc.poll() is None: | |
| proc.kill() | |
| _wait_exit(proc) | |
| def test_report_export_formats(isolated_lpu_home, config_auto): | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| for fmt in ("json", "csv", "html"): | |
| r = client.get("/api/report/export", params={"format": fmt}) | |
| assert r.status_code == 200, f"{fmt} export failed" | |
| path = r.json()["path"] | |
| assert Path(path).exists() and Path(path).stat().st_size > 0 | |
| # honesty note present in the JSON report | |
| rep = json.loads(Path(client.get("/api/report/export", | |
| params={"format": "json"}).json()["path"]) | |
| .read_text(encoding="utf-8")) | |
| assert "NEVER filled with estimates" in rep["honesty_note"] | |
| def test_dashboard_served(isolated_lpu_home, config_auto): | |
| app = create_app(config_auto) | |
| with TestClient(app) as client: | |
| r = client.get("/") | |
| assert r.status_code == 200 | |
| assert "LPU AI ACCELERATOR" in r.text | |
| assert "START ACCELERATOR" in r.text | |
| assert "EMERGENCY STOP" in r.text | |