timestral-coreai / tests /test_runtime.py
jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
9.53 kB
"""Native-process isolation and stage lifetime tests without model inference."""
import asyncio
from contextlib import asynccontextmanager
import os
from pathlib import Path
import subprocess
import sys
import types
import numpy as np
import pytest
from timestral_coreai import TimestralConfig, TimestralForecaster
from timestral_coreai.loading import STAGE_INPUTS, STAGE_OUTPUTS, load_model
from timestral_coreai.runtime import CoreAIRuntime
from timestral_coreai._worker import _run_graph
@pytest.fixture
def config():
return TimestralConfig(patch_size=4, d_model=16, num_heads=2, mlp_hidden_dim=32,
block_schedule=("time", "time"), quantile_knots=(.1, .5, .9),
block_size=2, context_length=16)
@pytest.fixture
def manifest():
return dict(stages=dict(embedding="embedding.aimodel", blocks=["block_000.aimodel", "block_001.aimodel"], head="head.aimodel"),
shape_limits=dict(max_rows=256, max_context_patches=1056, max_forecast_patches=1056))
@pytest.fixture(autouse=True)
def supported_platform(monkeypatch):
monkeypatch.setattr("timestral_coreai.runtime.platform.system", lambda: "Darwin")
monkeypatch.setattr("timestral_coreai.runtime.platform.machine", lambda: "arm64")
monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("27.0.1", (), ""))
def inputs():
return dict(scaled_patches=np.zeros((1, 2, 4), np.float32), observed_patches=np.ones((1, 2, 4), bool),
time_mask=np.ones((1, 1, 5, 5), bool), variate_mask=np.ones((5, 1, 1, 1), bool),
context_positions=np.zeros((1, 2), np.float32), forecast_positions=np.zeros((1, 3), np.float32))
def runtime(tmp_path, config, manifest, mode="ok", timeout=5):
log = tmp_path / "pids"
class Fake(CoreAIRuntime):
def _worker_command(self):
return [sys.executable, str(Path(__file__).with_name("fake_coreai_worker.py")), mode, str(log)]
return Fake(tmp_path, config=config, manifest=manifest, timeout_seconds=timeout), log
def assert_reaped(log):
for pid in log.read_text().splitlines():
with pytest.raises(ProcessLookupError):
os.kill(int(pid), 0)
def test_each_graph_gets_fresh_process_and_owned_array(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest)
with native:
first, second = native.run(inputs()), native.run(inputs())
assert first.shape == (1, 12, 3)
assert first.dtype == np.float32
assert first[0, 0, 0] != second[0, 0, 0]
assert native.worker_imports == {"torch": False, "mlx.core": False}
assert not native._slot
assert_reaped(log)
native.close()
with pytest.raises(RuntimeError, match="closed"):
native.run(inputs())
@pytest.mark.parametrize("mode,match", [("crash", "closed its pipe"), ("error", "injected stage error"),
("wrongdtype", "expected"), ("wrongshape", "expected"),
("wrongimports", "import boundary")])
def test_error_paths_reap_native_worker(tmp_path, config, manifest, mode, match):
native, log = runtime(tmp_path, config, manifest, mode)
with native:
with pytest.raises(RuntimeError, match=match):
native.run(inputs())
assert not native._slot
assert_reaped(log)
def test_timeout_reaps_native_worker(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest, "hang", 0.5)
with native:
with pytest.raises(TimeoutError, match="deadline"):
native.run(inputs())
assert not native._slot
assert_reaped(log)
def test_wrong_input_never_starts_worker(tmp_path, config, manifest):
native, log = runtime(tmp_path, config, manifest)
values = inputs()
values["time_mask"] = values["time_mask"].astype(np.float32)
with native, pytest.raises(ValueError, match="time_mask"):
native.run(values)
assert not log.exists()
def test_parent_pid_is_supplied_before_child_starts(tmp_path, config, manifest):
with CoreAIRuntime(tmp_path, config=config, manifest=manifest) as native:
assert native._worker_command()[-1] == str(os.getpid())
def test_orphaned_worker_rejects_mismatched_parent_before_native_loading(tmp_path):
process = subprocess.run([sys.executable, "-m", "timestral_coreai._worker", str(tmp_path), "1"],
capture_output=True, text=True, timeout=5)
assert process.returncode == 70
@pytest.mark.parametrize("policy", ["default", "gpu", "unknown"])
def test_only_cpu_policy_supported(tmp_path, config, manifest, policy):
with pytest.raises(ValueError, match="compute_unit='cpu'"):
CoreAIRuntime(tmp_path, config=config, manifest=manifest, compute_unit=policy)
with pytest.raises(ValueError, match="compute_unit='cpu'"):
load_model(tmp_path, compute_unit=policy)
def test_macos_version_checked_without_native_import(monkeypatch, tmp_path, config, manifest):
monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("26.0", (), ""))
with pytest.raises(RuntimeError, match="macOS 27"):
CoreAIRuntime(tmp_path, config=config, manifest=manifest)
def test_forecaster_forwards_cpu_policy(monkeypatch):
captured = {}
model = object()
def load(path, **kwargs):
captured.update(kwargs)
return model
class ForecasterDouble(TimestralForecaster):
def __init__(self, loaded, **kwargs):
self.model = loaded
self.kwargs = kwargs
monkeypatch.setattr("timestral_coreai.loading.load_model", load)
result = ForecasterDouble.from_pretrained("bundle", max_context=32)
assert captured["compute_unit"] == "cpu"
assert result.kwargs == {"max_context": 32}
@pytest.fixture
def native_double(monkeypatch):
state = {"active": 0, "maximum": 0, "calls": [], "fail": None, "supported": True}
policy = object()
class Array:
def __init__(self, data):
self.data = data
def numpy(self):
return self.data
class Asset:
def __init__(self, path):
self.name = path.name
self.output = None
@staticmethod
def load(path):
assert state["active"] == 0
return Asset(path)
@asynccontextmanager
async def executable(self, options):
assert options is policy
state["active"] += 1
state["maximum"] = max(state["maximum"], state["active"])
state["calls"].append(("open", self.name))
try:
yield self
finally:
if self.output is not None:
self.output.fill(np.nan)
state["active"] -= 1
state["calls"].append(("close", self.name))
def load_function(self, name):
assert name == "main"
async def run(values):
kind = "embedding" if self.name.startswith("embedding") else "head" if self.name.startswith("head") else "block"
assert set(values) == set(STAGE_INPUTS[kind])
if self.name == state["fail"]:
raise RuntimeError("injected stage failure")
if kind == "embedding":
self.output = np.ones((1, 5, 16), np.float32)
elif kind == "block":
self.output = values["hidden"].numpy().copy() + 1
else:
self.output = np.full((1, 12, 3), values["hidden"].numpy()[0, 0, 0], np.float32)
return {STAGE_OUTPUTS[kind]: Array(self.output)}
return run
class Options:
@staticmethod
def cpu_only():
return policy
@staticmethod
def is_supported():
return state["supported"]
for name, contents in {"coreai": {}, "coreai.authoring": {"AIModelAsset": Asset},
"coreai.runtime": {"NDArray": Array, "SpecializationOptions": Options}}.items():
module = types.ModuleType(name)
module.__dict__.update(contents)
monkeypatch.setitem(sys.modules, name, module)
return state
def test_native_stages_close_before_next_and_copy_outputs(native_double, tmp_path, config, manifest):
result = asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
np.testing.assert_array_equal(result, np.full((1, 12, 3), 3, np.float32))
assert native_double["maximum"] == 1
assert native_double["active"] == 0
assert native_double["calls"] == [event for name in ("embedding.aimodel", "block_000.aimodel", "block_001.aimodel", "head.aimodel")
for event in (("open", name), ("close", name))]
def test_native_stage_failure_releases_executable(native_double, tmp_path, config, manifest):
native_double["fail"] = "block_000.aimodel"
with pytest.raises(RuntimeError, match="injected stage"):
asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
assert native_double["active"] == 0
assert native_double["calls"][-1] == ("close", "block_000.aimodel")
def test_bundled_backend_cannot_silently_ignore_cpu_policy(native_double, tmp_path, config, manifest):
native_double["supported"] = False
with pytest.raises(RuntimeError, match="macOS CoreAI framework"):
asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
assert not native_double["calls"]