"""Bounded validation, report recovery, and reference-cache contracts without inference.""" import builtins import json import signal import sys import types import weakref import numpy as np import pytest from scripts import validate as validation from timestral_coreai import TimestralForecaster @pytest.fixture def sources(tmp_path): source, bundles = tmp_path / "source", tmp_path / "bundles" for base in (source, bundles): (base / "5M").mkdir(parents=True) (base / "5M" / "manifest.json").write_text("{}") (source / "timestral_mlx").mkdir() (source / "timestral_mlx" / "__init__.py").write_text("# reference source\n") return source, bundles @pytest.fixture def cases(): return [(name, np.array([1], np.float32), 1, {}) for name in validation.STANDARD_FIXTURES[:2]] def cache_case(cache, source, case, arrays=None, expected=None): arrays = arrays or {"mlx": np.ones((1, 1, 1, 3), np.float32)} validation.write_reference_cache(cache, "5M", case, arrays, validation.reference_provenance(source, "5M"), {"test": True}, expected or list(arrays)) def test_repeatable_fixture_selection_and_explicit_cpu(): args = validation.parse_args(["5M", "--fixture", validation.FIXTURE_NAMES[-1], "--fixture", validation.FIXTURE_NAMES[0], "--compute-unit", "cpu"]) selected = validation.select_cases(args.extended, args.fixture) assert [case[0] for case in selected] == [validation.FIXTURE_NAMES[-1], validation.FIXTURE_NAMES[0]] assert not args.keep_going and args.compute_unit == "cpu" assert selected[0][1].shape == (32768,) def test_cache_checks_inputs_sources_and_bytes(sources, cases, tmp_path): source, _ = sources cache = tmp_path / "cache" cache_case(cache, source, cases[0]) arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source) assert metadata["complete"] and arrays["mlx"].shape == (1, 1, 1, 3) changed = (cases[0][0], cases[0][1], 2, {}) with pytest.raises(ValueError, match="signature"): validation.load_reference_cache(cache, "5M", changed, source) model_source = source / "timestral_mlx" / "__init__.py" model_source.write_text("# changed source\n") with pytest.raises(ValueError, match="provenance"): validation.load_reference_cache(cache, "5M", cases[0], source) model_source.write_text("# reference source\n") array_file = cache / "5M" / f"{cases[0][0]}.npz" contents = bytearray(array_file.read_bytes()) contents[-1] ^= 1 array_file.write_bytes(contents) with pytest.raises(ValueError, match="integrity"): validation.load_reference_cache(cache, "5M", cases[0], source) def test_incomplete_cache_is_rejected(sources, cases, tmp_path): source, _ = sources cache = tmp_path / "cache" cache_case(cache, source, cases[0], expected=["mlx", "upstream"]) with pytest.raises(ValueError, match="Incomplete"): validation.load_reference_cache(cache, "5M", cases[0], source) @pytest.mark.parametrize("keep_going,expected_count", [(False, 1), (True, 2)]) def test_numerical_failure_checkpoints_and_stops_by_default( monkeypatch, sources, cases, tmp_path, keep_going, expected_count): source, bundles = sources cache, reports = tmp_path / "cache", tmp_path / "reports" for case in cases: cache_case(cache, source, case) monkeypatch.setattr(validation, "select_cases", lambda *args: cases) calls = [] class Candidate: model = types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="cpu", backend="macos-coreai")) def forecast(self, *args, **kwargs): checkpoint = json.loads((reports / "5M.json").read_text()) assert checkpoint["active_stage"] == "coreai_inference" calls.append("forecast") return types.SimpleNamespace(quantiles=np.full((1, 1, 1, 3), 2, np.float32)) def close(self): calls.append("close") monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: Candidate()) original_import = builtins.__import__ def no_reference_runtime(name, *args, **kwargs): if name == "torch" or name.startswith("torch.") or name == "mlx" or name.startswith("mlx."): raise AssertionError("cached validation imported a reference runtime") return original_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", no_reference_runtime) assert not validation.validate("5M", source, bundles, reports, selected=[c[0] for c in cases], keep_going=keep_going, compute_unit="cpu", reference_cache=cache) report = json.loads((reports / "5M.json").read_text()) assert len(report["fixtures"]) == expected_count assert calls == ["forecast"] * expected_count + ["close"] assert report["status"] == "failed" and not report["passed"] assert report["environment"]["backend"] == "macos-coreai" def test_subset_pass_is_not_full_suite_acceptance(monkeypatch, sources, cases, tmp_path): source, bundles = sources cache, reports = tmp_path / "cache", tmp_path / "reports" cache_case(cache, source, cases[0], arrays={ "mlx": np.ones((1, 1, 1, 3), np.float32), "upstream": np.ones((1, 1, 1, 3), np.float32), }) monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) candidate = types.SimpleNamespace( model=types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="default", backend="macos-coreai")), forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32)), close=lambda: None, ) monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate) assert validation.validate("5M", source, bundles, reports, selected=[cases[0][0]], reference_cache=cache) report = json.loads((reports / "5M.json").read_text()) assert report["passed"] and report["expected_fixture_count"] == 1 assert report["validation_scope"] == "selected_fixtures" assert not report["standard_suite_passed"] and not report["extended_suite_passed"] assert report["fixtures"][0]["upstream_comparison"]["passed"] def test_interruption_persists_incomplete_report(monkeypatch, sources, cases, tmp_path): source, bundles = sources cache, reports = tmp_path / "cache", tmp_path / "reports" cache_case(cache, source, cases[0]) monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) closed = [] def forecast(*args, **kwargs): raise validation.ValidationInterrupted(signal.SIGTERM) candidate = types.SimpleNamespace( model=types.SimpleNamespace(runtime=types.SimpleNamespace()), forecast=forecast, close=lambda: closed.append(True), ) monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate) with pytest.raises(validation.ValidationInterrupted): validation.validate("5M", source, bundles, reports, selected=[cases[0][0]], reference_cache=cache) report = json.loads((reports / "5M.json").read_text()) assert report["status"] == "interrupted" and not report["passed"] assert report["completed_fixture_count"] == 0 assert report["interrupted_fixture"] == cases[0][0] assert closed == [True] def test_reference_preparation_releases_mlx_before_torch(monkeypatch, sources, cases, tmp_path): source, _ = sources upstream = tmp_path / "upstream" (upstream / "timestral").mkdir(parents=True) (upstream / "timestral" / "__init__.py").write_text("# original source\n") cache = tmp_path / "cache" monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) refs = [] class MLX: def forecast(self, *args, **kwargs): return types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32)) def load_mlx(*args): model = MLX() refs.append(weakref.ref(model)) return model mlx = types.ModuleType("mlx") mlx.core = types.ModuleType("mlx.core") mlx.core.clear_cache = lambda: None reference_module = types.ModuleType("timestral_mlx") reference_module.TimestralForecaster = types.SimpleNamespace(from_pretrained=load_mlx) monkeypatch.setitem(sys.modules, "mlx", mlx) monkeypatch.setitem(sys.modules, "mlx.core", mlx.core) monkeypatch.setitem(sys.modules, "timestral_mlx", reference_module) def load_original(*args): assert refs[0]() is None, "MLX model was still alive when Torch model loaded" quantiles = types.SimpleNamespace(numpy=lambda: np.ones((1, 1, 1, 3), np.float32)) return types.SimpleNamespace(forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=quantiles)) monkeypatch.setattr(validation, "upstream_forecaster", load_original) validation.prepare_reference_cache("5M", source, cache, upstream, selected=[cases[0][0]]) arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source, upstream) assert set(arrays) == {"mlx", "upstream"} and metadata["complete"] assert json.loads((cache / "5M" / "prepare-status.json").read_text())["status"] == "completed"