File size: 9,526 Bytes
506f40d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
"""Native-process isolation and stage lifetime tests without model inference."""

import asyncio
from contextlib import asynccontextmanager
import os
from pathlib import Path
import subprocess
import sys
import types

import numpy as np
import pytest

from timestral_coreai import TimestralConfig, TimestralForecaster
from timestral_coreai.loading import STAGE_INPUTS, STAGE_OUTPUTS, load_model
from timestral_coreai.runtime import CoreAIRuntime
from timestral_coreai._worker import _run_graph


@pytest.fixture
def config():
    return TimestralConfig(patch_size=4, d_model=16, num_heads=2, mlp_hidden_dim=32,
                           block_schedule=("time", "time"), quantile_knots=(.1, .5, .9),
                           block_size=2, context_length=16)


@pytest.fixture
def manifest():
    return dict(stages=dict(embedding="embedding.aimodel", blocks=["block_000.aimodel", "block_001.aimodel"], head="head.aimodel"),
                shape_limits=dict(max_rows=256, max_context_patches=1056, max_forecast_patches=1056))


@pytest.fixture(autouse=True)
def supported_platform(monkeypatch):
    monkeypatch.setattr("timestral_coreai.runtime.platform.system", lambda: "Darwin")
    monkeypatch.setattr("timestral_coreai.runtime.platform.machine", lambda: "arm64")
    monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("27.0.1", (), ""))


def inputs():
    return dict(scaled_patches=np.zeros((1, 2, 4), np.float32), observed_patches=np.ones((1, 2, 4), bool),
                time_mask=np.ones((1, 1, 5, 5), bool), variate_mask=np.ones((5, 1, 1, 1), bool),
                context_positions=np.zeros((1, 2), np.float32), forecast_positions=np.zeros((1, 3), np.float32))


def runtime(tmp_path, config, manifest, mode="ok", timeout=5):
    log = tmp_path / "pids"

    class Fake(CoreAIRuntime):
        def _worker_command(self):
            return [sys.executable, str(Path(__file__).with_name("fake_coreai_worker.py")), mode, str(log)]

    return Fake(tmp_path, config=config, manifest=manifest, timeout_seconds=timeout), log


def assert_reaped(log):
    for pid in log.read_text().splitlines():
        with pytest.raises(ProcessLookupError):
            os.kill(int(pid), 0)


def test_each_graph_gets_fresh_process_and_owned_array(tmp_path, config, manifest):
    native, log = runtime(tmp_path, config, manifest)
    with native:
        first, second = native.run(inputs()), native.run(inputs())
        assert first.shape == (1, 12, 3)
        assert first.dtype == np.float32
        assert first[0, 0, 0] != second[0, 0, 0]
        assert native.worker_imports == {"torch": False, "mlx.core": False}
        assert not native._slot
        assert_reaped(log)
    native.close()
    with pytest.raises(RuntimeError, match="closed"):
        native.run(inputs())


@pytest.mark.parametrize("mode,match", [("crash", "closed its pipe"), ("error", "injected stage error"),
                                        ("wrongdtype", "expected"), ("wrongshape", "expected"),
                                        ("wrongimports", "import boundary")])
def test_error_paths_reap_native_worker(tmp_path, config, manifest, mode, match):
    native, log = runtime(tmp_path, config, manifest, mode)
    with native:
        with pytest.raises(RuntimeError, match=match):
            native.run(inputs())
        assert not native._slot
        assert_reaped(log)


def test_timeout_reaps_native_worker(tmp_path, config, manifest):
    native, log = runtime(tmp_path, config, manifest, "hang", 0.5)
    with native:
        with pytest.raises(TimeoutError, match="deadline"):
            native.run(inputs())
        assert not native._slot
        assert_reaped(log)


def test_wrong_input_never_starts_worker(tmp_path, config, manifest):
    native, log = runtime(tmp_path, config, manifest)
    values = inputs()
    values["time_mask"] = values["time_mask"].astype(np.float32)
    with native, pytest.raises(ValueError, match="time_mask"):
        native.run(values)
    assert not log.exists()


def test_parent_pid_is_supplied_before_child_starts(tmp_path, config, manifest):
    with CoreAIRuntime(tmp_path, config=config, manifest=manifest) as native:
        assert native._worker_command()[-1] == str(os.getpid())


def test_orphaned_worker_rejects_mismatched_parent_before_native_loading(tmp_path):
    process = subprocess.run([sys.executable, "-m", "timestral_coreai._worker", str(tmp_path), "1"],
                             capture_output=True, text=True, timeout=5)
    assert process.returncode == 70


@pytest.mark.parametrize("policy", ["default", "gpu", "unknown"])
def test_only_cpu_policy_supported(tmp_path, config, manifest, policy):
    with pytest.raises(ValueError, match="compute_unit='cpu'"):
        CoreAIRuntime(tmp_path, config=config, manifest=manifest, compute_unit=policy)
    with pytest.raises(ValueError, match="compute_unit='cpu'"):
        load_model(tmp_path, compute_unit=policy)


def test_macos_version_checked_without_native_import(monkeypatch, tmp_path, config, manifest):
    monkeypatch.setattr("timestral_coreai.runtime.platform.mac_ver", lambda: ("26.0", (), ""))
    with pytest.raises(RuntimeError, match="macOS 27"):
        CoreAIRuntime(tmp_path, config=config, manifest=manifest)


def test_forecaster_forwards_cpu_policy(monkeypatch):
    captured = {}
    model = object()

    def load(path, **kwargs):
        captured.update(kwargs)
        return model

    class ForecasterDouble(TimestralForecaster):
        def __init__(self, loaded, **kwargs):
            self.model = loaded
            self.kwargs = kwargs

    monkeypatch.setattr("timestral_coreai.loading.load_model", load)
    result = ForecasterDouble.from_pretrained("bundle", max_context=32)
    assert captured["compute_unit"] == "cpu"
    assert result.kwargs == {"max_context": 32}


@pytest.fixture
def native_double(monkeypatch):
    state = {"active": 0, "maximum": 0, "calls": [], "fail": None, "supported": True}
    policy = object()

    class Array:
        def __init__(self, data):
            self.data = data

        def numpy(self):
            return self.data

    class Asset:
        def __init__(self, path):
            self.name = path.name
            self.output = None

        @staticmethod
        def load(path):
            assert state["active"] == 0
            return Asset(path)

        @asynccontextmanager
        async def executable(self, options):
            assert options is policy
            state["active"] += 1
            state["maximum"] = max(state["maximum"], state["active"])
            state["calls"].append(("open", self.name))
            try:
                yield self
            finally:
                if self.output is not None:
                    self.output.fill(np.nan)
                state["active"] -= 1
                state["calls"].append(("close", self.name))

        def load_function(self, name):
            assert name == "main"

            async def run(values):
                kind = "embedding" if self.name.startswith("embedding") else "head" if self.name.startswith("head") else "block"
                assert set(values) == set(STAGE_INPUTS[kind])
                if self.name == state["fail"]:
                    raise RuntimeError("injected stage failure")
                if kind == "embedding":
                    self.output = np.ones((1, 5, 16), np.float32)
                elif kind == "block":
                    self.output = values["hidden"].numpy().copy() + 1
                else:
                    self.output = np.full((1, 12, 3), values["hidden"].numpy()[0, 0, 0], np.float32)
                return {STAGE_OUTPUTS[kind]: Array(self.output)}

            return run

    class Options:
        @staticmethod
        def cpu_only():
            return policy

        @staticmethod
        def is_supported():
            return state["supported"]

    for name, contents in {"coreai": {}, "coreai.authoring": {"AIModelAsset": Asset},
                           "coreai.runtime": {"NDArray": Array, "SpecializationOptions": Options}}.items():
        module = types.ModuleType(name)
        module.__dict__.update(contents)
        monkeypatch.setitem(sys.modules, name, module)
    return state


def test_native_stages_close_before_next_and_copy_outputs(native_double, tmp_path, config, manifest):
    result = asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
    np.testing.assert_array_equal(result, np.full((1, 12, 3), 3, np.float32))
    assert native_double["maximum"] == 1
    assert native_double["active"] == 0
    assert native_double["calls"] == [event for name in ("embedding.aimodel", "block_000.aimodel", "block_001.aimodel", "head.aimodel")
                                      for event in (("open", name), ("close", name))]


def test_native_stage_failure_releases_executable(native_double, tmp_path, config, manifest):
    native_double["fail"] = "block_000.aimodel"
    with pytest.raises(RuntimeError, match="injected stage"):
        asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
    assert native_double["active"] == 0
    assert native_double["calls"][-1] == ("close", "block_000.aimodel")


def test_bundled_backend_cannot_silently_ignore_cpu_policy(native_double, tmp_path, config, manifest):
    native_double["supported"] = False
    with pytest.raises(RuntimeError, match="macOS CoreAI framework"):
        asyncio.run(_run_graph(tmp_path, manifest, config, inputs()))
    assert not native_double["calls"]