File size: 4,243 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
"""V7 test fixtures: isolated LPU_HOME per test + fake trainer harness."""

from __future__ import annotations

import os
import sys
import time
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))


@pytest.fixture(autouse=True)
def isolated_lpu_home(tmp_path, monkeypatch):
    """Every test gets its own LPU_HOME so sessions/db/logs never collide."""
    home = tmp_path / "lpu_home"
    home.mkdir()
    monkeypatch.setenv("LPU_HOME", str(home))
    # fresh module state for anything caching the home at import time
    yield home


@pytest.fixture()
def config():
    return {
        "server": {"host": "127.0.0.1", "expose": False},
        "accelerator": {"autostart": False, "auto_connect": False,
                        "poll_seconds_idle": 0.2, "poll_seconds_active": 0.2},
        "optimizer": {"enabled": True, "auto_apply_level1": True,
                      "cooldown_s": 0.0, "max_proposals": 10},
        "bottleneck": {"window_s": 0.2, "min_samples": 3},
        "monitoring": {"enabled": False, "autostart": False},
        "integration": {"deep_scan": True},
    }


def run_fake_trainer(home: Path, iterations: int = 6, pause_poll: bool = True,
                     workers: int = 0, extra_settings: dict | None = None,
                     run_in_subprocess: bool = True):
    """Start a REAL python process that imports the REAL Integration SDK,
    runs a fake training loop with phase timings, registers a live parameter
    handler, and honors pause/stop. Returns the subprocess handle."""
    import subprocess

    code = f"""
import sys, os, time, json
sys.path.insert(0, {str(ROOT)!r})
os.environ["LPU_HOME"] = {str(home)!r}
from adapters.lpu_accelerate import LPUIntegration

extra = {extra_settings or dict()}
state = {{"workers": {workers}, "stop_after": None}}
integ = LPUIntegration(settings={{
    "num_workers": {workers}, "prefetch_factor": 1, "pin_memory": False,
    "persistent_workers": False, "micro_batch_size": 16,
    "gradient_accumulation": 1, "batch_size": 16,
    "train_images": 800, **extra}},
    project_root={str(ROOT)!r}, entry_script="fake_trainer.py")

def set_workers(v):
    state["workers"] = int(v)
    return True

integ.register_param("dataloader.num_workers", set_workers,
                     change_class="safe_boundary", current={workers})

for i in range({int(iterations)}):
    if not integ.pause_point():
        integ.checkpoint(phase_s=0.01, path="ckpt_final.pth")
        break
    t_data = 0.05 + 0.001 * i     # slow data phase on purpose (bottleneck)
    t_fwd = 0.01
    t_bwd = 0.02
    t_opt = 0.005
    time.sleep(0.02)
    integ.iteration(data_s=t_data, forward_s=t_fwd, backward_s=t_bwd,
                    optimizer_s=t_opt, h2d_s=0.002, images=16,
                    loss=0.5 - 0.01 * i, lr=3e-4)
integ.epoch_update(epoch=1, step={int(iterations)}, total_steps=100,
                   loss=0.4, psnr=31.62, ssim=0.8493, lr=3e-4, batch_size=16)
integ.validation(phase_s=0.5, psnr=31.62, ssim=0.8493)
integ.finalize(summary={{"final_psnr": 31.62}})
"""
    if run_in_subprocess:
        env = {**os.environ, "LPU_HOME": str(home),
               "PYTHONIOENCODING": "utf-8"}
        return subprocess.Popen(
            [sys.executable, "-c", code], env=env,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
    # in-process variant (used by direct SDK tests)
    exec(code, {"__name__": "__main__"})
    return None


def wait_for(path: Path, timeout: float = 10.0, contains: str = "") -> str:
    """Poll a file until it exists (and optionally contains text)."""
    deadline = time.time() + timeout
    last = ""
    while time.time() < deadline:
        try:
            last = path.read_text(encoding="utf-8")
            if not contains or contains in last:
                return last
        except OSError:
            pass
        time.sleep(0.1)
    return last


@pytest.fixture()
def fake_trainer(isolated_lpu_home):
    """Runs the subprocess trainer; yields (proc, home)."""
    import subprocess

    proc = run_fake_trainer(isolated_lpu_home)
    yield proc, isolated_lpu_home
    if proc.poll() is None:
        proc.kill()
        proc.wait(timeout=5)