"""Native-process isolation and stage lifetime tests without model inference.""" import asyncio from contextlib import asynccontextmanager import os from pathlib import Path import subprocess import sys import types import numpy as np import pytest from timestral_coreai import TimestralConfig, TimestralForecaster from timestral_coreai.loading import STAGE_INPUTS, STAGE_OUTPUTS, load_model from timestral_coreai.runtime import CoreAIRuntime from timestral_coreai._worker import _run_graph @pytest.fixture def config(): return TimestralConfig(patch_size=4, d_model=16, num_heads=2, mlp_hidden_dim=32, block_schedule=("time", "time"), quantile_knots=(.1, .5, .9), block_size=2, context_length=16) @pytest.fixture def manifest(): return dict(stages=dict(embedding="embedding.aimodel", blocks=["block_000.aimodel", "block_001.aimodel"], head="head.aimodel"), shape_limits=dict(max_rows=256, max_context_patches=1056, max_forecast_patches=1056)) @pytest.fixture(autouse=True) def supported_platform(monkeypatch): monkeypatch.setattr("timestral_coreai.runtime.platform.system", lambda: "Darwin") monkeypatch.setattr("timestral_coreai.runtime.platform.machine", lambda: "arm64") monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("27.0.1", (), "")) def inputs(): return dict(scaled_patches=np.zeros((1, 2, 4), np.float32), observed_patches=np.ones((1, 2, 4), bool), time_mask=np.ones((1, 1, 5, 5), bool), variate_mask=np.ones((5, 1, 1, 1), bool), context_positions=np.zeros((1, 2), np.float32), forecast_positions=np.zeros((1, 3), np.float32)) def runtime(tmp_path, config, manifest, mode="ok", timeout=5): log = tmp_path / "pids" class Fake(CoreAIRuntime): def _worker_command(self): return [sys.executable, str(Path(__file__).with_name("fake_coreai_worker.py")), mode, str(log)] return Fake(tmp_path, config=config, manifest=manifest, timeout_seconds=timeout), log def assert_reaped(log): for pid in log.read_text().splitlines(): with pytest.raises(ProcessLookupError): os.kill(int(pid), 0) def test_each_graph_gets_fresh_process_and_owned_array(tmp_path, config, manifest): native, log = runtime(tmp_path, config, manifest) with native: first, second = native.run(inputs()), native.run(inputs()) assert first.shape == (1, 12, 3) assert first.dtype == np.float32 assert first[0, 0, 0] != second[0, 0, 0] assert native.worker_imports == {"torch": False, "mlx.core": False} assert not native._slot assert_reaped(log) native.close() with pytest.raises(RuntimeError, match="closed"): native.run(inputs()) @pytest.mark.parametrize("mode,match", [("crash", "closed its pipe"), ("error", "injected stage error"), ("wrongdtype", "expected"), ("wrongshape", "expected"), ("wrongimports", "import boundary")]) def test_error_paths_reap_native_worker(tmp_path, config, manifest, mode, match): native, log = runtime(tmp_path, config, manifest, mode) with native: with pytest.raises(RuntimeError, match=match): native.run(inputs()) assert not native._slot assert_reaped(log) def test_timeout_reaps_native_worker(tmp_path, config, manifest): native, log = runtime(tmp_path, config, manifest, "hang", 0.5) with native: with pytest.raises(TimeoutError, match="deadline"): native.run(inputs()) assert not native._slot assert_reaped(log) def test_wrong_input_never_starts_worker(tmp_path, config, manifest): native, log = runtime(tmp_path, config, manifest) values = inputs() values["time_mask"] = values["time_mask"].astype(np.float32) with native, pytest.raises(ValueError, match="time_mask"): native.run(values) assert not log.exists() def test_parent_pid_is_supplied_before_child_starts(tmp_path, config, manifest): with CoreAIRuntime(tmp_path, config=config, manifest=manifest) as native: assert native._worker_command()[-1] == str(os.getpid()) def test_orphaned_worker_rejects_mismatched_parent_before_native_loading(tmp_path): process = subprocess.run([sys.executable, "-m", "timestral_coreai._worker", str(tmp_path), "1"], capture_output=True, text=True, timeout=5) assert process.returncode == 70 @pytest.mark.parametrize("policy", ["default", "gpu", "unknown"]) def test_only_cpu_policy_supported(tmp_path, config, manifest, policy): with pytest.raises(ValueError, match="compute_unit='cpu'"): CoreAIRuntime(tmp_path, config=config, manifest=manifest, compute_unit=policy) with pytest.raises(ValueError, match="compute_unit='cpu'"): load_model(tmp_path, compute_unit=policy) def test_macos_version_checked_without_native_import(monkeypatch, tmp_path, config, manifest): monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("26.0", (), "")) with pytest.raises(RuntimeError, match="macOS 27"): CoreAIRuntime(tmp_path, config=config, manifest=manifest) def test_forecaster_forwards_cpu_policy(monkeypatch): captured = {} model = object() def load(path, **kwargs): captured.update(kwargs) return model class ForecasterDouble(TimestralForecaster): def __init__(self, loaded, **kwargs): self.model = loaded self.kwargs = kwargs monkeypatch.setattr("timestral_coreai.loading.load_model", load) result = ForecasterDouble.from_pretrained("bundle", max_context=32) assert captured["compute_unit"] == "cpu" assert result.kwargs == {"max_context": 32} @pytest.fixture def native_double(monkeypatch): state = {"active": 0, "maximum": 0, "calls": [], "fail": None, "supported": True} policy = object() class Array: def __init__(self, data): self.data = data def numpy(self): return self.data class Asset: def __init__(self, path): self.name = path.name self.output = None @staticmethod def load(path): assert state["active"] == 0 return Asset(path) @asynccontextmanager async def executable(self, options): assert options is policy state["active"] += 1 state["maximum"] = max(state["maximum"], state["active"]) state["calls"].append(("open", self.name)) try: yield self finally: if self.output is not None: self.output.fill(np.nan) state["active"] -= 1 state["calls"].append(("close", self.name)) def load_function(self, name): assert name == "main" async def run(values): kind = "embedding" if self.name.startswith("embedding") else "head" if self.name.startswith("head") else "block" assert set(values) == set(STAGE_INPUTS[kind]) if self.name == state["fail"]: raise RuntimeError("injected stage failure") if kind == "embedding": self.output = np.ones((1, 5, 16), np.float32) elif kind == "block": self.output = values["hidden"].numpy().copy() + 1 else: self.output = np.full((1, 12, 3), values["hidden"].numpy()[0, 0, 0], np.float32) return {STAGE_OUTPUTS[kind]: Array(self.output)} return run class Options: @staticmethod def cpu_only(): return policy @staticmethod def is_supported(): return state["supported"] for name, contents in {"coreai": {}, "coreai.authoring": {"AIModelAsset": Asset}, "coreai.runtime": {"NDArray": Array, "SpecializationOptions": Options}}.items(): module = types.ModuleType(name) module.__dict__.update(contents) monkeypatch.setitem(sys.modules, name, module) return state def test_native_stages_close_before_next_and_copy_outputs(native_double, tmp_path, config, manifest): result = asyncio.run(_run_graph(tmp_path, manifest, config, inputs())) np.testing.assert_array_equal(result, np.full((1, 12, 3), 3, np.float32)) assert native_double["maximum"] == 1 assert native_double["active"] == 0 assert native_double["calls"] == [event for name in ("embedding.aimodel", "block_000.aimodel", "block_001.aimodel", "head.aimodel") for event in (("open", name), ("close", name))] def test_native_stage_failure_releases_executable(native_double, tmp_path, config, manifest): native_double["fail"] = "block_000.aimodel" with pytest.raises(RuntimeError, match="injected stage"): asyncio.run(_run_graph(tmp_path, manifest, config, inputs())) assert native_double["active"] == 0 assert native_double["calls"][-1] == ("close", "block_000.aimodel") def test_bundled_backend_cannot_silently_ignore_cpu_policy(native_double, tmp_path, config, manifest): native_double["supported"] = False with pytest.raises(RuntimeError, match="macOS CoreAI framework"): asyncio.run(_run_graph(tmp_path, manifest, config, inputs())) assert not native_double["calls"]