File size: 9,526 Bytes
506f40d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 | """Native-process isolation and stage lifetime tests without model inference."""
import asyncio
from contextlib import asynccontextmanager
import os
from pathlib import Path
import subprocess
import sys
import types
import numpy as np
import pytest
from timestral_coreai import TimestralConfig, TimestralForecaster
from timestral_coreai.loading import STAGE_INPUTS, STAGE_OUTPUTS, load_model
from timestral_coreai.runtime import CoreAIRuntime
from timestral_coreai._worker import _run_graph
@pytest.fixture
def config():
return TimestralConfig(patch_size=4, d_model=16, num_heads=2, mlp_hidden_dim=32,
block_schedule=("time", "time"), quantile_knots=(.1, .5, .9),
block_size=2, context_length=16)
@pytest.fixture
def manifest():
return dict(stages=dict(embedding="embedding.aimodel", blocks=["block_000.aimodel", "block_001.aimodel"], head="head.aimodel"),
shape_limits=dict(max_rows=256, max_context_patches=1056, max_forecast_patches=1056))
@pytest.fixture(autouse=True)
def supported_platform(monkeypatch):
monkeypatch.setattr("timestral_coreai.runtime.platform.system", lambda: "Darwin")
monkeypatch.setattr("timestral_coreai.runtime.platform.machine", lambda: "arm64")
monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("27.0.1", (), ""))
def inputs():
return dict(scaled_patches=np.zeros((1, 2, 4), np.float32), observed_patches=np.ones((1, 2, 4), bool),
time_mask=np.ones((1, 1, 5, 5), bool), variate_mask=np.ones((5, 1, 1, 1), bool),
context_positions=np.zeros((1, 2), np.float32), forecast_positions=np.zeros((1, 3), np.float32))
def runtime(tmp_path, config, manifest, mode="ok", timeout=5):
log = tmp_path / "pids"
class Fake(CoreAIRuntime):
def _worker_command(self):
return [sys.executable, str(Path(__file__).with_name("fake_coreai_worker.py")), mode, str(log)]
return Fake(tmp_path, config=config, manifest=manifest, timeout_seconds=timeout), log
def assert_reaped(log):
for pid in log.read_text().splitlines():
with pytest.raises(ProcessLookupError):
os.kill(int(pid), 0)
def test_each_graph_gets_fresh_process_and_owned_array(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest)
with native:
first, second = native.run(inputs()), native.run(inputs())
assert first.shape == (1, 12, 3)
assert first.dtype == np.float32
assert first[0, 0, 0] != second[0, 0, 0]
assert native.worker_imports == {"torch": False, "mlx.core": False}
assert not native._slot
assert_reaped(log)
native.close()
with pytest.raises(RuntimeError, match="closed"):
native.run(inputs())
@pytest.mark.parametrize("mode,match", [("crash", "closed its pipe"), ("error", "injected stage error"),
("wrongdtype", "expected"), ("wrongshape", "expected"),
("wrongimports", "import boundary")])
def test_error_paths_reap_native_worker(tmp_path, config, manifest, mode, match):
native, log = runtime(tmp_path, config, manifest, mode)
with native:
with pytest.raises(RuntimeError, match=match):
native.run(inputs())
assert not native._slot
assert_reaped(log)
def test_timeout_reaps_native_worker(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest, "hang", 0.5)
with native:
with pytest.raises(TimeoutError, match="deadline"):
native.run(inputs())
assert not native._slot
assert_reaped(log)
def test_wrong_input_never_starts_worker(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest)
values = inputs()
values["time_mask"] = values["time_mask"].astype(np.float32)
with native, pytest.raises(ValueError, match="time_mask"):
native.run(values)
assert not log.exists()
def test_parent_pid_is_supplied_before_child_starts(tmp_path, config, manifest):
with CoreAIRuntime(tmp_path, config=config, manifest=manifest) as native:
assert native._worker_command()[-1] == str(os.getpid())
def test_orphaned_worker_rejects_mismatched_parent_before_native_loading(tmp_path):
process = subprocess.run([sys.executable, "-m", "timestral_coreai._worker", str(tmp_path), "1"],
capture_output=True, text=True, timeout=5)
assert process.returncode == 70
@pytest.mark.parametrize("policy", ["default", "gpu", "unknown"])
def test_only_cpu_policy_supported(tmp_path, config, manifest, policy):
with pytest.raises(ValueError, match="compute_unit='cpu'"):
CoreAIRuntime(tmp_path, config=config, manifest=manifest, compute_unit=policy)
with pytest.raises(ValueError, match="compute_unit='cpu'"):
load_model(tmp_path, compute_unit=policy)
def test_macos_version_checked_without_native_import(monkeypatch, tmp_path, config, manifest):
monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("26.0", (), ""))
with pytest.raises(RuntimeError, match="macOS 27"):
CoreAIRuntime(tmp_path, config=config, manifest=manifest)
def test_forecaster_forwards_cpu_policy(monkeypatch):
captured = {}
model = object()
def load(path, **kwargs):
captured.update(kwargs)
return model
class ForecasterDouble(TimestralForecaster):
def __init__(self, loaded, **kwargs):
self.model = loaded
self.kwargs = kwargs
monkeypatch.setattr("timestral_coreai.loading.load_model", load)
result = ForecasterDouble.from_pretrained("bundle", max_context=32)
assert captured["compute_unit"] == "cpu"
assert result.kwargs == {"max_context": 32}
@pytest.fixture
def native_double(monkeypatch):
state = {"active": 0, "maximum": 0, "calls": [], "fail": None, "supported": True}
policy = object()
class Array:
def __init__(self, data):
self.data = data
def numpy(self):
return self.data
class Asset:
def __init__(self, path):
self.name = path.name
self.output = None
@staticmethod
def load(path):
assert state["active"] == 0
return Asset(path)
@asynccontextmanager
async def executable(self, options):
assert options is policy
state["active"] += 1
state["maximum"] = max(state["maximum"], state["active"])
state["calls"].append(("open", self.name))
try:
yield self
finally:
if self.output is not None:
self.output.fill(np.nan)
state["active"] -= 1
state["calls"].append(("close", self.name))
def load_function(self, name):
assert name == "main"
async def run(values):
kind = "embedding" if self.name.startswith("embedding") else "head" if self.name.startswith("head") else "block"
assert set(values) == set(STAGE_INPUTS[kind])
if self.name == state["fail"]:
raise RuntimeError("injected stage failure")
if kind == "embedding":
self.output = np.ones((1, 5, 16), np.float32)
elif kind == "block":
self.output = values["hidden"].numpy().copy() + 1
else:
self.output = np.full((1, 12, 3), values["hidden"].numpy()[0, 0, 0], np.float32)
return {STAGE_OUTPUTS[kind]: Array(self.output)}
return run
class Options:
@staticmethod
def cpu_only():
return policy
@staticmethod
def is_supported():
return state["supported"]
for name, contents in {"coreai": {}, "coreai.authoring": {"AIModelAsset": Asset},
"coreai.runtime": {"NDArray": Array, "SpecializationOptions": Options}}.items():
module = types.ModuleType(name)
module.__dict__.update(contents)
monkeypatch.setitem(sys.modules, name, module)
return state
def test_native_stages_close_before_next_and_copy_outputs(native_double, tmp_path, config, manifest):
result = asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
np.testing.assert_array_equal(result, np.full((1, 12, 3), 3, np.float32))
assert native_double["maximum"] == 1
assert native_double["active"] == 0
assert native_double["calls"] == [event for name in ("embedding.aimodel", "block_000.aimodel", "block_001.aimodel", "head.aimodel")
for event in (("open", name), ("close", name))]
def test_native_stage_failure_releases_executable(native_double, tmp_path, config, manifest):
native_double["fail"] = "block_000.aimodel"
with pytest.raises(RuntimeError, match="injected stage"):
asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
assert native_double["active"] == 0
assert native_double["calls"][-1] == ("close", "block_000.aimodel")
def test_bundled_backend_cannot_silently_ignore_cpu_policy(native_double, tmp_path, config, manifest):
native_double["supported"] = False
with pytest.raises(RuntimeError, match="macOS CoreAI framework"):
asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
assert not native_double["calls"]
|