Virtual-LPU / server /app.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
20.6 kB
"""LPU AI Accelerator V7 — independent FastAPI server (mission §2.C, §13).
Runs SEPARATELY from VS Code and from the training process. Binding rules:
* default host 127.0.0.1 (loopback ONLY) - the API controls processes,
so it must not be network-exposed by default
* binding to anything else requires config server.expose=true AND a token
(generated at first start, stored under <lpu_home>/server_token, never
written to logs); requests then need `Authorization: Bearer <token>`
* sensitive operations (emergency stop, level-3 apply) additionally
require explicit consent flags in the request body
Endpoints (all read-only unless marked):
GET /api/status accelerator + runtime snapshot
GET /api/hardware GPU/CPU/RAM/disk snapshot (?force=1 resample)
GET /api/projects detected processes
GET /api/project attached project analysis
POST /api/detect run discovery now
POST /api/connect {pid|session_id}
POST /api/disconnect
POST /api/monitoring/start /api/monitoring/stop
POST /api/accelerator/start /api/accelerator/stop
POST /api/training/pause /api/training/resume (SDK-only)
POST /api/training/stop {mode: graceful|emergency, consent:bool}
GET /api/terminal event history (?kind=&level=&search=&limit=)
GET /api/sessions session history (SQLite)
GET /api/params parameter governance table
GET /api/proposals optimizer proposals
POST /api/proposals/{id}/approve | /reject
POST /api/proposals/{id}/apply {confirm:bool, override_validation:bool}
GET /api/changes change ledger
POST /api/changes/{id}/rollback
GET /api/bottleneck current classification + evidence
GET /api/report/export?format=json|csv|html
WS /ws live events stream
GET / dashboard (self-contained HTML)
"""
from __future__ import annotations
import asyncio
import json
import os
import secrets
import sys
import threading
import time
import uuid
from pathlib import Path
from typing import Optional
from fastapi import FastAPI, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse
from lpu.version import __version__ as LPU_VERSION
from lpu.accelerator import AcceleratorService
from lpu.accelerator.bus import EventBus
from lpu.accelerator.parameters import classify_for_dashboard
from lpu.accelerator.storage import TelemetryStorage
def create_app(config: Optional[dict] = None) -> FastAPI:
cfg = config or {}
srv = cfg.get("server", {})
host = str(srv.get("host", "127.0.0.1"))
expose = bool(srv.get("expose", False))
bus = EventBus()
storage = TelemetryStorage()
service = AcceleratorService(config=cfg, bus=bus, storage=storage)
# Managed training processes launched by this Desktop instance.
_launch_lock = threading.RLock()
_launch_jobs = {}
token = None
if host not in ("127.0.0.1", "localhost", "::1") and expose:
token_path = service.home / "server_token"
if token_path.exists():
token = token_path.read_text(encoding="utf-8").strip()
else:
token = secrets.token_urlsafe(32)
token_path.parent.mkdir(parents=True, exist_ok=True)
token_path.write_text(token, encoding="utf-8")
try:
token_path.chmod(0o600)
except OSError:
pass
app = FastAPI(title="LPU AI Accelerator V7", version=LPU_VERSION,
docs_url=None, redoc_url=None)
# ------------------------------------------------------------ auth
@app.middleware("http")
async def _auth(request: Request, call_next):
if token is not None:
path = request.url.path
if path.startswith("/api") or path == "/ws":
auth = request.headers.get("authorization", "")
provided = auth[7:].strip() if auth.lower().startswith("bearer ") else ""
key = request.query_params.get("token", "")
if provided != token and key != token:
return JSONResponse({"error": "authentication required"}, 401)
return await call_next(request)
def _require_consent(body: dict, field: str = "consent") -> None:
if not body.get(field):
raise HTTPException(400, f"'{field}' must be explicitly true for this operation")
# ------------------------------------------------------------ status
@app.get("/api/status")
async def status():
return service.status()
@app.get("/api/hardware")
async def hardware(force: bool = False):
return service.hardware(force=force)
@app.get("/api/projects")
async def projects():
return {"processes": service.detect_projects()}
@app.post("/api/detect")
async def detect():
return {"processes": service.detect_projects()}
@app.get("/api/project")
async def project():
st = service.status()
return st.get("detected_project") or {"connected": False}
@app.post("/api/connect")
async def connect(body: dict = None):
body = body or {}
res = service.connect(pid=body.get("pid"), session_id=body.get("session_id"))
if not res.get("ok"):
raise HTTPException(400, res.get("error", "connect failed"))
return res
@app.post("/api/disconnect")
async def disconnect():
return service.disconnect()
@app.post("/api/monitoring/start")
async def mon_start():
return service.start_monitoring()
@app.post("/api/monitoring/stop")
async def mon_stop():
return service.stop_monitoring()
@app.post("/api/accelerator/start")
async def acc_start():
res = service.start()
if not res.get("ok"):
raise HTTPException(500, "accelerator start failed")
return res
@app.post("/api/accelerator/stop")
async def acc_stop():
return service.stop()
# ------------------------------------------------------------ project launch
@app.post("/api/launch")
async def launch_project(body: dict = None):
body = body or {}
# Validate script
script_value = body.get("script")
if not isinstance(script_value, str) or not script_value.strip():
raise HTTPException(400, "'script' is required")
script = Path(script_value).expanduser()
try:
script = script.resolve(strict=True)
except (OSError, RuntimeError):
raise HTTPException(400, "Script not found")
if not script.is_file() or script.suffix.lower() != ".py":
raise HTTPException(400, "A valid Python script is required")
# Validate arguments
args = body.get("args", [])
if not isinstance(args, list) or not all(
isinstance(x, str) for x in args
):
raise HTTPException(400, "args must be a list of strings")
# Working directory
cwd_value = body.get("cwd")
cwd = Path(cwd_value).expanduser() if cwd_value else script.parent
try:
cwd = cwd.resolve(strict=True)
except (OSError, RuntimeError):
raise HTTPException(400, "Working directory not found")
if not cwd.is_dir():
raise HTTPException(400, "Invalid working directory")
# Resolve model interpreter
integration_cfg = cfg.get("integration", {})
if not isinstance(integration_cfg, dict):
integration_cfg = {}
python_exe = (
body.get("python")
or integration_cfg.get("python")
or os.environ.get("LPU_PYTHON")
or r"C:\ipara\Scripts\python.exe"
)
if not isinstance(python_exe, str) or not python_exe.strip():
raise HTTPException(400, "Invalid Python interpreter")
if not Path(python_exe).is_file():
raise HTTPException(
400,
f"Python interpreter not found: {python_exe}"
)
# Prevent duplicate launch of the same script
script_key = str(script).casefold()
with _launch_lock:
for existing in _launch_jobs.values():
if (
existing["script"].casefold() == script_key
and existing["state"] in ("starting", "running")
):
raise HTTPException(
409,
"This script is already running"
)
job_id = uuid.uuid4().hex
job = {
"job_id": job_id,
"state": "starting",
"script": str(script),
"python": python_exe,
"cwd": str(cwd),
"pid": None,
"session_id": None,
"created_ts": time.time(),
"returncode": None,
"result": None,
"error": None,
"_proc": None,
"_stop_requested": False,
"_stop_event": threading.Event(),
}
_launch_jobs[job_id] = job
# Called immediately after Popen succeeds
def on_started(info):
with _launch_lock:
current = _launch_jobs.get(job_id)
if current is None:
return
current.update({
"state": "running",
"pid": info["pid"],
"session_id": info["session_id"],
"python": info["python"],
"_proc": info["proc"],
})
# Background worker: launcher.launch is blocking,
# so it must never run inside the FastAPI request thread.
def worker():
try:
from lpu.integration.launcher import launch
stop_event = job["_stop_event"]
report = launch(
argv=[str(script), *args],
python=python_exe,
config=cfg,
hooks=bool(body.get("hooks", True)),
cwd=str(cwd),
on_started=on_started,
stop_event=stop_event,
)
stopped = stop_event.is_set()
with _launch_lock:
current = _launch_jobs[job_id]
current.update({
"state": "stopped" if stopped else "finished",
"returncode": report.returncode,
"result": report.summary(),
"_proc": None,
})
except Exception as exc:
with _launch_lock:
current = _launch_jobs[job_id]
current.update({
"state": "failed",
"error": f"{type(exc).__name__}: {exc}",
"_proc": None,
})
thread = threading.Thread(
target=worker,
name=f"lpu-project-{job_id[:8]}",
daemon=True,
)
with _launch_lock:
job["_thread"] = thread
thread.start()
return {
"ok": True,
"job": {
k: v for k, v in job.items()
if not k.startswith("_")
},
}
# List launched projects
@app.get("/api/launches")
async def list_launches():
with _launch_lock:
jobs = [
{
k: v for k, v in job.items()
if not k.startswith("_")
}
for job in _launch_jobs.values()
]
return {"launches": jobs}
# Stop a project launched by this LPU instance
@app.post("/api/launch/{job_id}/stop")
async def stop_launch(job_id: str):
with _launch_lock:
job = _launch_jobs.get(job_id)
if job is None:
raise HTTPException(404, "Unknown launch job")
if job["state"] not in ("starting", "running"):
return {
"ok": True,
"state": job["state"],
"already_finished": True,
}
job["_stop_requested"] = True
stop_event = job["_stop_event"]
# Request graceful shutdown through launcher.
stop_event.set()
return {
"ok": True,
"job_id": job_id,
"stop_requested": True,
}
# ------------------------------------------------------------ terminal
@app.get("/api/terminal")
async def terminal(kind: Optional[str] = None, level: Optional[str] = None,
search: str = "", limit: int = 500):
# memory buffer first (sub-ms), then durable SQLite history so the
# terminal survives dashboard/server restarts (mission §5/§10);
# de-dup by (seq, ts, text) since both layers hold the same event
mem = bus.snapshot(limit=limit, kind=kind, level=level, search=search)
try:
disk = storage.list_events(kind=kind, level=level, search=search,
limit=limit)
except Exception: # noqa: BLE001 - history is best-effort
disk = []
seen = {(e.get("seq"), e.get("ts"), e.get("text")) for e in mem}
merged = mem + [e for e in disk
if (e.get("seq"), e.get("ts"), e.get("text")) not in seen]
merged.sort(key=lambda e: e.get("ts") or 0)
return {"events": merged[-limit:]}
@app.get("/api/sessions")
async def sessions():
return {"sessions": storage.list_sessions()}
# ------------------------------------------------------------ optimizer
@app.get("/api/params")
async def params():
return {"parameters": classify_for_dashboard()}
@app.get("/api/proposals")
async def proposals():
return {"proposals": service.optimizer.proposals_view()}
@app.post("/api/proposals/{pid}/approve")
async def approve(pid: str):
p = service.optimizer.approve(pid)
if p is None:
raise HTTPException(404, "unknown proposal")
return {"ok": True, "proposal": p.as_dict()}
@app.post("/api/proposals/{pid}/reject")
async def reject(pid: str):
p = service.optimizer.reject(pid)
if p is None:
raise HTTPException(404, "unknown proposal")
return {"ok": True, "proposal": p.as_dict()}
@app.post("/api/proposals/{pid}/apply")
async def apply(pid: str, body: dict = None):
body = body or {}
res = service.optimizer.apply(
pid, user_confirmed=bool(body.get("confirm")),
override_validation=bool(body.get("override_validation")))
if not res.get("ok"):
raise HTTPException(409, res.get("error", res.get("detail", "apply failed")))
service.emit("optimizer", "info",
f"[OPTIMIZER] Configuration Change Applied: "
f"{pid} -> {res.get('status')}")
return res
@app.get("/api/changes")
async def changes():
return {"changes": storage.changes()}
@app.post("/api/changes/{cid}/rollback")
async def rollback(cid: str):
res = service.optimizer.rollback(cid)
if not res.get("ok"):
raise HTTPException(409, res.get("error", "rollback failed"))
service.emit("optimizer", "info",
f"[OPTIMIZER] Rollback executed: {cid} -> {res.get('status')}")
return res
@app.get("/api/bottleneck")
async def bottleneck():
return service.bottleneck.summary()
# ------------------------------------------------------------ reports
@app.get("/api/report/export")
async def report_export(format: str = Query("json", pattern="^(json|csv|html)$")):
rep = storage.build_report(service.status())
out_dir = service.home / "reports"
try:
path = storage.export_report(rep, format, out_dir)
except OSError as exc:
raise HTTPException(500, f"report write failed: {exc}")
return {"path": str(path), "format": format,
"note": "report contains only measured data; unmeasured fields "
"are labeled, never invented"}
# ------------------------------------------------------------ websocket
@app.websocket("/ws")
async def ws_endpoint(ws: WebSocket):
await ws.accept()
loop = asyncio.get_running_loop()
queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
dropped = threading.Event()
def _on_event(event: dict) -> None:
def _put():
try:
queue.put_nowait(event)
except asyncio.QueueFull:
dropped.set()
loop.call_soon_threadsafe(_put)
unsubscribe = bus.subscribe(_on_event)
try:
# initial snapshot so a reconnecting dashboard isn't blank
for ev in bus.snapshot(limit=200):
await ws.send_text(json.dumps(ev, default=str))
while True:
try:
event = await asyncio.wait_for(queue.get(), timeout=15.0)
except asyncio.TimeoutError:
await ws.send_text(json.dumps({"kind": "heartbeat", "level": "info",
"text": "", "ts": __import__("time").time()}))
continue
await ws.send_text(json.dumps(event, default=str))
if dropped.is_set():
await ws.send_text(json.dumps(
{"kind": "lpu", "level": "warn",
"text": "event queue overflow: some terminal lines were "
"dropped for THIS websocket subscriber "
"(storage retains all)", "ts": __import__("time").time()}))
dropped.clear()
except WebSocketDisconnect:
pass
finally:
unsubscribe()
# ------------------------------------------------------------ dashboard
def _find_dashboard() -> Optional[Path]:
"""dashboard.html in the source tree OR inside a PyInstaller bundle
(--onefile data files live under sys._MEIPASS)."""
candidates = [Path(__file__).resolve().parent.parent / "frontend"
/ "dashboard.html"]
meipass = getattr(sys, "_MEIPASS", None)
if meipass:
m = Path(meipass)
candidates += [m / "frontend" / "dashboard.html",
m.parent / "frontend" / "dashboard.html"]
for cand in candidates:
if cand.exists():
return cand
return None
_DASH = _find_dashboard()
@app.get("/")
async def dashboard():
if _DASH.exists():
return FileResponse(_DASH, media_type="text/html")
return PlainTextResponse("dashboard.html not found", status_code=404)
@app.on_event("startup")
async def _startup():
if cfg.get("accelerator", {}).get("autostart", True):
service.start()
@app.on_event("shutdown")
async def _shutdown():
# STOP the service; training processes are NEVER touched here
service.stop()
storage.close()
return app
def main() -> int: # pragma: no cover - manual entry
import argparse
import uvicorn
parser = argparse.ArgumentParser(prog="lpu-server",
description="LPU AI Accelerator V7 server")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8710)
parser.add_argument("--config", default=None, help="path to lpu_config_v7.yaml")
args = parser.parse_args()
cfg = {}
if args.config:
import yaml
cfg = yaml.safe_load(Path(args.config).read_text(encoding="utf-8")) or {}
if args.host != "127.0.0.1":
cfg.setdefault("server", {})["expose"] = True
cfg["server"]["host"] = args.host
app = create_app(cfg)
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())