File size: 14,594 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
"""Test 04 — End-to-end: real server + real SDK fake-trainer subprocess.

Boots the FastAPI app with TestClient, starts a REAL python process running
the REAL Integration SDK, and verifies the full loop:
detect -> connect -> telemetry -> bottleneck -> proposal -> boundary apply
-> rollback -> pause -> graceful stop.
"""

from __future__ import annotations

import json
import sys
import time

import pytest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from fastapi.testclient import TestClient

from server.app import create_app
from tests.conftest import run_fake_trainer, wait_for


@pytest.fixture()
def config_auto(config):
    """E2E needs the service actually starting with monitoring active."""
    cfg = dict(config)
    cfg["accelerator"] = {**cfg["accelerator"], "autostart": True,
                          "auto_connect": False}
    cfg["monitoring"] = {**cfg.get("monitoring", {}),
                         "enabled": False, "autostart": True}
    return cfg


def _wait_exit(proc, timeout=15):
    try:
        out, err = proc.communicate(timeout=timeout)
    except Exception:  # noqa: BLE001
        proc.kill()
        out, err = proc.communicate()
    return out, err


def test_full_control_loop(isolated_lpu_home, config_auto):
    app = create_app(config_auto)
    with TestClient(app) as client:
        # 1) status: server up, accelerator started by app startup event
        st = client.get("/api/status").json()
        assert st["accelerator_status"] == "running"
        assert st["runtime_version"].startswith("7."), st["runtime_version"]

        # 2) hardware snapshot exists (CPU/RAM real; GPU None-honest here)
        hw = client.get("/api/hardware", params={"force": "1"}).json()
        assert hw["ram"]["total_gb"] > 0

        # 3) launch the fake trainer with the real SDK
        proc = run_fake_trainer(isolated_lpu_home, iterations=600)
        try:
            # 4) session descriptor appears
            sessions_root = isolated_lpu_home / "sessions"
            deadline = time.time() + 10
            sid = None
            while time.time() < deadline:
                if sessions_root.is_dir():
                    for d in sessions_root.iterdir():
                        if (d / "session.json").exists():
                            desc = json.loads((d / "session.json").read_text(encoding="utf-8"))
                            if desc.get("sdk"):
                                sid = desc["session_id"]
                                break
                if sid:
                    break
                time.sleep(0.1)
            assert sid, "SDK session descriptor must appear"
            desc = json.loads((sessions_root / sid / "session.json")
                              .read_text(encoding="utf-8"))
            assert desc["sdk"] is True and desc["pid"] == proc.pid

            # 5) connect via session id
            res = client.post("/api/connect", json={"session_id": sid}).json()
            assert res["ok"] is True
            st = client.get("/api/status").json()
            assert st["connected"] is True
            assert st["training"]["attached"] is True
            assert st["training"]["session_id"] == sid

            # 6) telemetry flows: wait for metrics + status with phase means
            deadline = time.time() + 15
            phases = None
            while time.time() < deadline:
                st = client.get("/api/status").json()
                phases = (st.get("training") or {}).get("phases_mean_s")
                if phases:
                    break
                time.sleep(0.3)
            assert phases, "phase timings must reach the service from the SDK"
            assert "data" in phases and "forward" in phases
            st = client.get("/api/status").json()
            assert st["training"]["data_wait_ratio"] > 0.5, \
                "fake trainer's slow data phase must surface as data-wait"

            # 7) terminal shows real events
            ev = client.get("/api/terminal").json()["events"]
            assert any(e["kind"] == "project" for e in ev)

            # 8) bottleneck must classify (slow data + data-wait >= 0.5)
            deadline = time.time() + 20
            bn = None
            while time.time() < deadline:
                bn = client.get("/api/bottleneck").json()["current"]
                if bn and "DataLoader" in (bn or {}).get("category", ""):
                    break
                time.sleep(0.3)
            assert bn, "bottleneck engine must produce a report"
            assert "DataLoader" in bn["category"]
            assert bn["confidence"] in ("Measured",)

            # 9) optimizer produced data-path proposals from the bottleneck
            deadline = time.time() + 20
            props = []
            while time.time() < deadline:
                props = client.get("/api/proposals").json()["proposals"]
                if props:
                    break
                time.sleep(0.3)
            assert props, "proposals must be derived from measured evidence"
            prop = next(p for p in props
                        if p["param"] == "dataloader.num_workers")
            assert prop["level"] == 1
            assert prop["evidence"]["data_wait_ratio"] >= 0.5

            # 10) apply the proposal (level 1, auto-apply enabled in config)
            res = client.post(f"/api/proposals/{prop['id']}/apply",
                              json={"confirm": True}).json()
            assert res["status"] in ("applied", "queued_at_boundary")
            # the fake trainer only applies at boundary() -> queued expected
            assert res["status"] == "queued_at_boundary"

            # 11) trainer crosses the boundary at its next pause_point()
            deadline = time.time() + 15
            ack_path = sessions_root / sid / "acks.jsonl"
            applied_ok = False
            while time.time() < deadline:
                if ack_path.exists():
                    for line in ack_path.read_text(encoding="utf-8").splitlines():
                        line = line.strip()
                        if not line:
                            continue
                        try:
                            a = json.loads(line)
                        except json.JSONDecodeError:
                            continue
                        if (a.get("cmd") == "set_parameter"
                                and a.get("param") == "dataloader.num_workers"
                                and a.get("status") == "applied"):
                            applied_ok = True
                            break
                if applied_ok:
                    break
                time.sleep(0.2)
            assert applied_ok, "SDK must ack the boundary application"

            # 12) ledger records the change
            changes = client.get("/api/changes").json()["changes"]
            assert any(c["param"] == "dataloader.num_workers" for c in changes)

            # 13) status settings reflect the LIVE value
            deadline = time.time() + 10
            live = None
            while time.time() < deadline:
                st = client.get("/api/status").json()
                settings = ((st.get("training") or {}).get("settings") or {})
                live = settings.get("dataloader.num_workers", settings.get("num_workers"))
                if live == 4:
                    break
                time.sleep(0.3)
            assert live == 4, "live settings must show the applied value"

            # 14) pause / resume through the API
            res = client.post("/api/training/pause").json()
            assert res["ok"] is True
            deadline = time.time() + 8
            while time.time() < deadline:
                st = client.get("/api/status").json()
                if (st.get("training") or {}).get("state") == "paused":
                    break
                time.sleep(0.2)
            assert (client.get("/api/status").json()
                    .get("training", {}).get("state")) == "paused"
            res = client.post("/api/training/resume").json()
            assert res["ok"] is True

            # 15) rollback the change
            changes = client.get("/api/changes").json()["changes"]
            ch = next(c for c in changes
                      if c["param"] == "dataloader.num_workers"
                      and c["status"] in ("applied", "queued_at_boundary"))
            res = client.post(f"/api/changes/{ch['id']}/rollback").json()
            assert res["ok"] is True

            # 16) graceful stop: trainer exits by itself at its boundary
            res = client.post("/api/training/stop",
                              json={"mode": "graceful"}).json()
            assert res["ok"] is True
            out, err = _wait_exit(proc, timeout=20)
            # trainer finished cleanly (finalize writes status finished)
            st_file = sessions_root / sid / "status.json"
            final = json.loads(wait_for(st_file, timeout=10))
            assert final["state"] == "finished"
            assert final.get("final_psnr") == 31.62
            assert proc.returncode in (0, None)
        finally:
            if proc.poll() is None:
                proc.kill()
                _wait_exit(proc)


def test_websocket_stream(isolated_lpu_home, config_auto):
    import subprocess
    script = isolated_lpu_home / "fake_train_edsrc.py"
    script.write_text("import time\ntime.sleep(30)\n", encoding="utf-8")
    proc = subprocess.Popen([sys.executable, str(script)])
    try:
        app = create_app(config_auto)
        with TestClient(app) as client:
            with client.websocket_connect("/ws") as ws:
                # snapshot drains first; a discovered training-like process
                # must then stream live project events
                deadline = time.time() + 10
                got = None
                client.post("/api/detect")
                while time.time() < deadline:
                    msg = json.loads(ws.receive_text())
                    if msg.get("kind") == "project":
                        got = msg
                        break
                assert got is not None, "discovery events must stream over WebSocket"
        # disconnect mid-stream must not break the bus for later subscribers
            with client.websocket_connect("/ws") as ws2:
                client.post("/api/monitoring/stop")
                msg = json.loads(ws2.receive_text())
                assert msg["kind"] in ("lpu", "heartbeat", "project")
    finally:
        if proc.poll() is None:
            proc.kill()
            proc.wait(timeout=5)


def test_accelerator_stop_leaves_training_alone(isolated_lpu_home, config_auto):
    """Mission rule: STOP ACCELERATOR never stops training."""
    app = create_app(config_auto)
    with TestClient(app) as client:
        proc = run_fake_trainer(isolated_lpu_home, iterations=400)
        try:
            time.sleep(1.0)
            assert proc.poll() is None, "trainer must be running"
            res = client.post("/api/accelerator/stop").json()
            assert res["ok"] is True
            time.sleep(1.5)
            assert proc.poll() is None, \
                "trainer must STILL be running after accelerator stop"
            # and the server itself is still up
            st = client.get("/api/status").json()
            assert st["accelerator_status"] == "stopped"
            st2 = client.post("/api/accelerator/start").json()
            assert st2["ok"] is True
        finally:
            if proc.poll() is None:
                client.post("/api/training/stop", json={"mode": "emergency",
                                                        "consent": True})
                _wait_exit(proc, timeout=10)
            if proc.poll() is None:
                proc.kill()
                _wait_exit(proc)


def test_emergency_stop_requires_consent(isolated_lpu_home, config_auto):
    app = create_app(config_auto)
    with TestClient(app) as client:
        proc = run_fake_trainer(isolated_lpu_home, iterations=400)
        try:
            # attach to the trainer's SDK session first (emergency stop acts
            # on the ATTACHED process)
            deadline = time.time() + 10
            sid = None
            root = isolated_lpu_home / "sessions"
            while time.time() < deadline:
                if root.is_dir():
                    for d in root.iterdir():
                        if (d / "session.json").exists():
                            sid = d.name
                            break
                if sid:
                    break
                time.sleep(0.1)
            assert sid
            assert client.post("/api/connect", json={"session_id": sid}).json()["ok"]
            time.sleep(0.3)
            # without consent: 400
            r = client.post("/api/training/stop",
                            json={"mode": "emergency", "consent": False})
            assert r.status_code == 400
            assert proc.poll() is None, "no consent -> process untouched"
            # with consent: terminated
            r = client.post("/api/training/stop",
                            json={"mode": "emergency", "consent": True})
            assert r.status_code == 200
            _wait_exit(proc, timeout=10)
            assert proc.poll() is not None
        finally:
            if proc.poll() is None:
                proc.kill()
                _wait_exit(proc)


def test_report_export_formats(isolated_lpu_home, config_auto):
    app = create_app(config_auto)
    with TestClient(app) as client:
        for fmt in ("json", "csv", "html"):
            r = client.get("/api/report/export", params={"format": fmt})
            assert r.status_code == 200, f"{fmt} export failed"
            path = r.json()["path"]
            assert Path(path).exists() and Path(path).stat().st_size > 0
        # honesty note present in the JSON report
        rep = json.loads(Path(client.get("/api/report/export",
                                         params={"format": "json"}).json()["path"])
                         .read_text(encoding="utf-8"))
        assert "NEVER filled with estimates" in rep["honesty_note"]


def test_dashboard_served(isolated_lpu_home, config_auto):
    app = create_app(config_auto)
    with TestClient(app) as client:
        r = client.get("/")
        assert r.status_code == 200
        assert "LPU AI ACCELERATOR" in r.text
        assert "START ACCELERATOR" in r.text
        assert "EMERGENCY STOP" in r.text