Virtual-LPU / tests /test_01_sdk.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
8.18 kB
"""Test 01 — Integration SDK core contracts (mission §6, §11, §12).
Runs the REAL SDK in-process against an isolated LPU_HOME:
- session descriptor / status / metrics / acks files appear as specified
- set_parameter: unknown param REFUSED; unregistered param REFUSED honestly
- safe-boundary param queued -> applied only at boundary()
- hot-reload param applied immediately
- request_pause / request_resume / graceful_stop honored at pause_point()
- fail-open: disabled SDK never raises
"""
from __future__ import annotations
import json
import time
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from adapters.lpu_accelerate import LPUIntegration
from lpu.ipc import SessionChannel
from lpu.ipc.protocol import Envelope
def _sessions(root: Path):
return root / "sessions"
def test_sdk_writes_session_layout(isolated_lpu_home):
integ = LPUIntegration(settings={"num_workers": 0}, project_root="/tmp/proj")
root = _sessions(isolated_lpu_home) / integ.session_id
assert (root / "session.json").exists()
desc = json.loads((root / "session.json").read_text(encoding="utf-8"))
assert desc["sdk"] is True
assert desc["pid"] > 0
assert desc["settings"] == {"num_workers": 0}
# status written at construction
st = json.loads((root / "status.json").read_text(encoding="utf-8"))
assert st["state"] in ("starting", "training")
integ.finalize()
def test_sdk_iteration_and_metrics(isolated_lpu_home):
integ = LPUIntegration(settings={"batch_size": 16})
integ.iteration(data_s=0.5, forward_s=0.1, backward_s=0.2,
optimizer_s=0.05, images=16, loss=0.42, lr=3e-4)
time.sleep(0.1)
metrics = (_sessions(isolated_lpu_home) / integ.session_id / "metrics.jsonl") \
.read_text(encoding="utf-8").strip().splitlines()
assert metrics, "metrics.jsonl must have the iteration record"
rec = json.loads(metrics[-1])
assert rec["data"] == 0.5 and rec["images"] == 16
integ.finalize()
def test_set_parameter_unknown_refused(isolated_lpu_home):
integ = LPUIntegration(settings={})
ch = SessionChannel(session_id=integ.session_id,
root=_sessions(isolated_lpu_home) / integ.session_id)
ch.send_command("set_parameter", {
"cmd_id": "t1", "param": "training.learning_rate", "value": 1e-3,
"boundary": True})
deadline = time.time() + 5
acks = ""
while time.time() < deadline:
acks = _read(integ, "acks.jsonl")
if "refused" in acks:
break
time.sleep(0.1)
assert "refused" in acks, "unregistered parameter must be refused honestly"
assert "no registered handler" in acks
integ.finalize()
def _read(integ, name) -> str:
try:
return ((Path(integ.root) / name)).read_text(encoding="utf-8")
except OSError:
return ""
def test_safe_boundary_param_queued_then_applied(isolated_lpu_home):
applied = []
integ = LPUIntegration(settings={"num_workers": 0})
integ.register_param("dataloader.num_workers",
lambda v: (applied.append(v), True)[1],
change_class="safe_boundary", current=0)
ch = SessionChannel(session_id=integ.session_id,
root=_sessions(isolated_lpu_home) / integ.session_id)
ch.send_command("set_parameter", {
"cmd_id": "t2", "param": "dataloader.num_workers", "value": 4,
"boundary": True})
deadline = time.time() + 5
while time.time() < deadline and "queued_at_boundary" not in _read(integ, "acks.jsonl"):
time.sleep(0.1)
assert "queued_at_boundary" in _read(integ, "acks.jsonl")
assert applied == [], "safe-boundary value must NOT apply before boundary()"
integ.boundary()
assert applied == [4], "value must apply exactly once at the boundary"
ack_lines = [json.loads(l) for l in _read(integ, "acks.jsonl").splitlines() if l.strip()]
t2_acks = [a for a in ack_lines if a.get("cmd_id") == "t2"]
assert any(a.get("status") == "applied" and a.get("boundary") is True
for a in t2_acks), f"expected an applied boundary ack, got {t2_acks}"
assert integ.live_settings["dataloader.num_workers"] == 4
integ.finalize()
def test_hot_reload_param_applied_immediately(isolated_lpu_home):
applied = []
integ = LPUIntegration(settings={"rate": 1})
integ.register_param("runtime.monitoring_rate_s",
lambda v: (applied.append(v), True)[1],
change_class="hot_reload", current=1)
ch = SessionChannel(session_id=integ.session_id,
root=_sessions(isolated_lpu_home) / integ.session_id)
ch.send_command("set_parameter", {
"cmd_id": "t3", "param": "runtime.monitoring_rate_s", "value": 2,
"boundary": False})
deadline = time.time() + 5
while time.time() < deadline and not applied:
time.sleep(0.05)
assert applied == [2], "hot-reload param applies without a boundary"
assert integ.live_settings["runtime.monitoring_rate_s"] == 2
integ.finalize()
def test_pause_resume_and_graceful_stop(isolated_lpu_home):
integ = LPUIntegration(settings={})
ch = SessionChannel(session_id=integ.session_id,
root=_sessions(isolated_lpu_home) / integ.session_id)
# pause
ch.send_command("request_pause", {"cmd_id": "p1"})
deadline = time.time() + 5
while time.time() < deadline and not integ.paused:
time.sleep(0.05)
assert integ.paused, "SDK must honor request_pause"
# resume
ch.send_command("request_resume", {"cmd_id": "r1"})
deadline = time.time() + 5
while time.time() < deadline and integ.paused:
time.sleep(0.05)
assert not integ.paused
# graceful stop
ch.send_command("graceful_stop", {"cmd_id": "g1"})
deadline = time.time() + 5
while time.time() < deadline and not integ.should_stop:
time.sleep(0.05)
assert integ.should_stop, "SDK must honor graceful_stop"
assert integ.pause_point(timeout_s=1.0) is False, \
"pause_point returns False after a graceful stop (trainer exits)"
st = json.loads(_read(integ, "status.json"))
assert st["state"] in ("stopping", "finished")
integ.finalize()
def test_unknown_command_refused(isolated_lpu_home):
integ = LPUIntegration(settings={})
# write a raw envelope with an unknown command directly
env = Envelope(type="command", session_id=integ.session_id,
payload={"cmd": "format_disk", "cmd_id": "bad1"})
path = Path(integ.root) / "commands.jsonl"
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "a", encoding="utf-8") as fh:
fh.write(env.to_json() + "\n")
deadline = time.time() + 5
while time.time() < deadline and "unknown command" not in _read(integ, "acks.jsonl"):
time.sleep(0.1)
assert "unknown command" in _read(integ, "acks.jsonl"), \
"commands outside the allowlist are refused, never executed"
integ.finalize()
def test_sdk_fail_open_when_disabled(isolated_lpu_home, monkeypatch):
monkeypatch.setenv("LPU_SDK_ENABLED", "0")
integ = LPUIntegration(settings={})
assert integ.enabled is False
# none of these may raise, even with the whole machinery off
integ.iteration(data_s=1.0)
integ.epoch_update(epoch=1)
integ.boundary()
integ.finalize()
def test_sdk_survives_unwritable_home(monkeypatch, tmp_path):
# point LPU_HOME somewhere impossible -> SDK disables itself, no crash
monkeypatch.setenv("LPU_HOME", str(tmp_path / "no" / "such" / "dir" / "x"))
monkeypatch.setenv("LPU_SDK_ENABLED", "1")
integ = LPUIntegration(settings={})
# mkdir parents=True will create it; force failure differently: a FILE
# as the home directory
blocker = tmp_path / "blocker"
blocker.write_text("x", encoding="utf-8")
monkeypatch.setenv("LPU_HOME", str(blocker / "sessions"))
integ2 = LPUIntegration(settings={})
assert integ2.enabled in (True, False) # whatever happened, no exception
integ2.iteration(data_s=0.1) # still safe
integ2.finalize()