Spaces:
Running
Running
Download tests/conftest.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 4.24 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/tests/conftest.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/tests/conftest.py
-
curl -L -o conftest.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/tests/conftest.py
4.24 kB
| """V7 test fixtures: isolated LPU_HOME per test + fake trainer harness.""" | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import pytest | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| def isolated_lpu_home(tmp_path, monkeypatch): | |
| """Every test gets its own LPU_HOME so sessions/db/logs never collide.""" | |
| home = tmp_path / "lpu_home" | |
| home.mkdir() | |
| monkeypatch.setenv("LPU_HOME", str(home)) | |
| # fresh module state for anything caching the home at import time | |
| yield home | |
| def config(): | |
| return { | |
| "server": {"host": "127.0.0.1", "expose": False}, | |
| "accelerator": {"autostart": False, "auto_connect": False, | |
| "poll_seconds_idle": 0.2, "poll_seconds_active": 0.2}, | |
| "optimizer": {"enabled": True, "auto_apply_level1": True, | |
| "cooldown_s": 0.0, "max_proposals": 10}, | |
| "bottleneck": {"window_s": 0.2, "min_samples": 3}, | |
| "monitoring": {"enabled": False, "autostart": False}, | |
| "integration": {"deep_scan": True}, | |
| } | |
| def run_fake_trainer(home: Path, iterations: int = 6, pause_poll: bool = True, | |
| workers: int = 0, extra_settings: dict | None = None, | |
| run_in_subprocess: bool = True): | |
| """Start a REAL python process that imports the REAL Integration SDK, | |
| runs a fake training loop with phase timings, registers a live parameter | |
| handler, and honors pause/stop. Returns the subprocess handle.""" | |
| import subprocess | |
| code = f""" | |
| import sys, os, time, json | |
| sys.path.insert(0, {str(ROOT)!r}) | |
| os.environ["LPU_HOME"] = {str(home)!r} | |
| from adapters.lpu_accelerate import LPUIntegration | |
| extra = {extra_settings or dict()} | |
| state = {{"workers": {workers}, "stop_after": None}} | |
| integ = LPUIntegration(settings={{ | |
| "num_workers": {workers}, "prefetch_factor": 1, "pin_memory": False, | |
| "persistent_workers": False, "micro_batch_size": 16, | |
| "gradient_accumulation": 1, "batch_size": 16, | |
| "train_images": 800, **extra}}, | |
| project_root={str(ROOT)!r}, entry_script="fake_trainer.py") | |
| def set_workers(v): | |
| state["workers"] = int(v) | |
| return True | |
| integ.register_param("dataloader.num_workers", set_workers, | |
| change_class="safe_boundary", current={workers}) | |
| for i in range({int(iterations)}): | |
| if not integ.pause_point(): | |
| integ.checkpoint(phase_s=0.01, path="ckpt_final.pth") | |
| break | |
| t_data = 0.05 + 0.001 * i # slow data phase on purpose (bottleneck) | |
| t_fwd = 0.01 | |
| t_bwd = 0.02 | |
| t_opt = 0.005 | |
| time.sleep(0.02) | |
| integ.iteration(data_s=t_data, forward_s=t_fwd, backward_s=t_bwd, | |
| optimizer_s=t_opt, h2d_s=0.002, images=16, | |
| loss=0.5 - 0.01 * i, lr=3e-4) | |
| integ.epoch_update(epoch=1, step={int(iterations)}, total_steps=100, | |
| loss=0.4, psnr=31.62, ssim=0.8493, lr=3e-4, batch_size=16) | |
| integ.validation(phase_s=0.5, psnr=31.62, ssim=0.8493) | |
| integ.finalize(summary={{"final_psnr": 31.62}}) | |
| """ | |
| if run_in_subprocess: | |
| env = {**os.environ, "LPU_HOME": str(home), | |
| "PYTHONIOENCODING": "utf-8"} | |
| return subprocess.Popen( | |
| [sys.executable, "-c", code], env=env, | |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | |
| # in-process variant (used by direct SDK tests) | |
| exec(code, {"__name__": "__main__"}) | |
| return None | |
| def wait_for(path: Path, timeout: float = 10.0, contains: str = "") -> str: | |
| """Poll a file until it exists (and optionally contains text).""" | |
| deadline = time.time() + timeout | |
| last = "" | |
| while time.time() < deadline: | |
| try: | |
| last = path.read_text(encoding="utf-8") | |
| if not contains or contains in last: | |
| return last | |
| except OSError: | |
| pass | |
| time.sleep(0.1) | |
| return last | |
| def fake_trainer(isolated_lpu_home): | |
| """Runs the subprocess trainer; yields (proc, home).""" | |
| import subprocess | |
| proc = run_fake_trainer(isolated_lpu_home) | |
| yield proc, isolated_lpu_home | |
| if proc.poll() is None: | |
| proc.kill() | |
| proc.wait(timeout=5) | |