timestral-coreai / tests /test_validation_tools.py
jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
9.37 kB
"""Bounded validation, report recovery, and reference-cache contracts without inference."""
import builtins
import json
import signal
import sys
import types
import weakref
import numpy as np
import pytest
from scripts import validate as validation
from timestral_coreai import TimestralForecaster
@pytest.fixture
def sources(tmp_path):
source, bundles = tmp_path / "source", tmp_path / "bundles"
for base in (source, bundles):
(base / "5M").mkdir(parents=True)
(base / "5M" / "manifest.json").write_text("{}")
(source / "timestral_mlx").mkdir()
(source / "timestral_mlx" / "__init__.py").write_text("# reference source\n")
return source, bundles
@pytest.fixture
def cases():
return [(name, np.array([1], np.float32), 1, {})
for name in validation.STANDARD_FIXTURES[:2]]
def cache_case(cache, source, case, arrays=None, expected=None):
arrays = arrays or {"mlx": np.ones((1, 1, 1, 3), np.float32)}
validation.write_reference_cache(cache, "5M", case, arrays,
validation.reference_provenance(source, "5M"),
{"test": True}, expected or list(arrays))
def test_repeatable_fixture_selection_and_explicit_cpu():
args = validation.parse_args(["5M", "--fixture", validation.FIXTURE_NAMES[-1],
"--fixture", validation.FIXTURE_NAMES[0], "--compute-unit", "cpu"])
selected = validation.select_cases(args.extended, args.fixture)
assert [case[0] for case in selected] == [validation.FIXTURE_NAMES[-1], validation.FIXTURE_NAMES[0]]
assert not args.keep_going and args.compute_unit == "cpu"
assert selected[0][1].shape == (32768,)
def test_cache_checks_inputs_sources_and_bytes(sources, cases, tmp_path):
source, _ = sources
cache = tmp_path / "cache"
cache_case(cache, source, cases[0])
arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source)
assert metadata["complete"] and arrays["mlx"].shape == (1, 1, 1, 3)
changed = (cases[0][0], cases[0][1], 2, {})
with pytest.raises(ValueError, match="signature"):
validation.load_reference_cache(cache, "5M", changed, source)
model_source = source / "timestral_mlx" / "__init__.py"
model_source.write_text("# changed source\n")
with pytest.raises(ValueError, match="provenance"):
validation.load_reference_cache(cache, "5M", cases[0], source)
model_source.write_text("# reference source\n")
array_file = cache / "5M" / f"{cases[0][0]}.npz"
contents = bytearray(array_file.read_bytes())
contents[-1] ^= 1
array_file.write_bytes(contents)
with pytest.raises(ValueError, match="integrity"):
validation.load_reference_cache(cache, "5M", cases[0], source)
def test_incomplete_cache_is_rejected(sources, cases, tmp_path):
source, _ = sources
cache = tmp_path / "cache"
cache_case(cache, source, cases[0], expected=["mlx", "upstream"])
with pytest.raises(ValueError, match="Incomplete"):
validation.load_reference_cache(cache, "5M", cases[0], source)
@pytest.mark.parametrize("keep_going,expected_count", [(False, 1), (True, 2)])
def test_numerical_failure_checkpoints_and_stops_by_default(
monkeypatch, sources, cases, tmp_path, keep_going, expected_count):
source, bundles = sources
cache, reports = tmp_path / "cache", tmp_path / "reports"
for case in cases:
cache_case(cache, source, case)
monkeypatch.setattr(validation, "select_cases", lambda *args: cases)
calls = []
class Candidate:
model = types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="cpu", backend="macos-coreai"))
def forecast(self, *args, **kwargs):
checkpoint = json.loads((reports / "5M.json").read_text())
assert checkpoint["active_stage"] == "coreai_inference"
calls.append("forecast")
return types.SimpleNamespace(quantiles=np.full((1, 1, 1, 3), 2, np.float32))
def close(self):
calls.append("close")
monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: Candidate())
original_import = builtins.__import__
def no_reference_runtime(name, *args, **kwargs):
if name == "torch" or name.startswith("torch.") or name == "mlx" or name.startswith("mlx."):
raise AssertionError("cached validation imported a reference runtime")
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", no_reference_runtime)
assert not validation.validate("5M", source, bundles, reports, selected=[c[0] for c in cases],
keep_going=keep_going, compute_unit="cpu", reference_cache=cache)
report = json.loads((reports / "5M.json").read_text())
assert len(report["fixtures"]) == expected_count
assert calls == ["forecast"] * expected_count + ["close"]
assert report["status"] == "failed" and not report["passed"]
assert report["environment"]["backend"] == "macos-coreai"
def test_subset_pass_is_not_full_suite_acceptance(monkeypatch, sources, cases, tmp_path):
source, bundles = sources
cache, reports = tmp_path / "cache", tmp_path / "reports"
cache_case(cache, source, cases[0], arrays={
"mlx": np.ones((1, 1, 1, 3), np.float32),
"upstream": np.ones((1, 1, 1, 3), np.float32),
})
monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1])
candidate = types.SimpleNamespace(
model=types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="default", backend="macos-coreai")),
forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32)),
close=lambda: None,
)
monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate)
assert validation.validate("5M", source, bundles, reports,
selected=[cases[0][0]], reference_cache=cache)
report = json.loads((reports / "5M.json").read_text())
assert report["passed"] and report["expected_fixture_count"] == 1
assert report["validation_scope"] == "selected_fixtures"
assert not report["standard_suite_passed"] and not report["extended_suite_passed"]
assert report["fixtures"][0]["upstream_comparison"]["passed"]
def test_interruption_persists_incomplete_report(monkeypatch, sources, cases, tmp_path):
source, bundles = sources
cache, reports = tmp_path / "cache", tmp_path / "reports"
cache_case(cache, source, cases[0])
monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1])
closed = []
def forecast(*args, **kwargs):
raise validation.ValidationInterrupted(signal.SIGTERM)
candidate = types.SimpleNamespace(
model=types.SimpleNamespace(runtime=types.SimpleNamespace()), forecast=forecast,
close=lambda: closed.append(True),
)
monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate)
with pytest.raises(validation.ValidationInterrupted):
validation.validate("5M", source, bundles, reports, selected=[cases[0][0]], reference_cache=cache)
report = json.loads((reports / "5M.json").read_text())
assert report["status"] == "interrupted" and not report["passed"]
assert report["completed_fixture_count"] == 0
assert report["interrupted_fixture"] == cases[0][0]
assert closed == [True]
def test_reference_preparation_releases_mlx_before_torch(monkeypatch, sources, cases, tmp_path):
source, _ = sources
upstream = tmp_path / "upstream"
(upstream / "timestral").mkdir(parents=True)
(upstream / "timestral" / "__init__.py").write_text("# original source\n")
cache = tmp_path / "cache"
monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1])
refs = []
class MLX:
def forecast(self, *args, **kwargs):
return types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32))
def load_mlx(*args):
model = MLX()
refs.append(weakref.ref(model))
return model
mlx = types.ModuleType("mlx")
mlx.core = types.ModuleType("mlx.core")
mlx.core.clear_cache = lambda: None
reference_module = types.ModuleType("timestral_mlx")
reference_module.TimestralForecaster = types.SimpleNamespace(from_pretrained=load_mlx)
monkeypatch.setitem(sys.modules, "mlx", mlx)
monkeypatch.setitem(sys.modules, "mlx.core", mlx.core)
monkeypatch.setitem(sys.modules, "timestral_mlx", reference_module)
def load_original(*args):
assert refs[0]() is None, "MLX model was still alive when Torch model loaded"
quantiles = types.SimpleNamespace(numpy=lambda: np.ones((1, 1, 1, 3), np.float32))
return types.SimpleNamespace(forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=quantiles))
monkeypatch.setattr(validation, "upstream_forecaster", load_original)
validation.prepare_reference_cache("5M", source, cache, upstream, selected=[cases[0][0]])
arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source, upstream)
assert set(arrays) == {"mlx", "upstream"} and metadata["complete"]
assert json.loads((cache / "5M" / "prepare-status.json").read_text())["status"] == "completed"