Download tests/test_validation_tools.py from jbilcke/timestral-coreai: direct link, hf CLI and curl.
- Browser
- Download file 9.37 kB
-
https://huggingface.co/jbilcke/timestral-coreai/resolve/main/tests/test_validation_tools.py
- Command line
-
hf download hf://jbilcke/timestral-coreai/tests/test_validation_tools.py
-
curl -L -o test_validation_tools.py https://huggingface.co/jbilcke/timestral-coreai/resolve/main/tests/test_validation_tools.py
9.37 kB
| """Bounded validation, report recovery, and reference-cache contracts without inference.""" | |
| import builtins | |
| import json | |
| import signal | |
| import sys | |
| import types | |
| import weakref | |
| import numpy as np | |
| import pytest | |
| from scripts import validate as validation | |
| from timestral_coreai import TimestralForecaster | |
| def sources(tmp_path): | |
| source, bundles = tmp_path / "source", tmp_path / "bundles" | |
| for base in (source, bundles): | |
| (base / "5M").mkdir(parents=True) | |
| (base / "5M" / "manifest.json").write_text("{}") | |
| (source / "timestral_mlx").mkdir() | |
| (source / "timestral_mlx" / "__init__.py").write_text("# reference source\n") | |
| return source, bundles | |
| def cases(): | |
| return [(name, np.array([1], np.float32), 1, {}) | |
| for name in validation.STANDARD_FIXTURES[:2]] | |
| def cache_case(cache, source, case, arrays=None, expected=None): | |
| arrays = arrays or {"mlx": np.ones((1, 1, 1, 3), np.float32)} | |
| validation.write_reference_cache(cache, "5M", case, arrays, | |
| validation.reference_provenance(source, "5M"), | |
| {"test": True}, expected or list(arrays)) | |
| def test_repeatable_fixture_selection_and_explicit_cpu(): | |
| args = validation.parse_args(["5M", "--fixture", validation.FIXTURE_NAMES[-1], | |
| "--fixture", validation.FIXTURE_NAMES[0], "--compute-unit", "cpu"]) | |
| selected = validation.select_cases(args.extended, args.fixture) | |
| assert [case[0] for case in selected] == [validation.FIXTURE_NAMES[-1], validation.FIXTURE_NAMES[0]] | |
| assert not args.keep_going and args.compute_unit == "cpu" | |
| assert selected[0][1].shape == (32768,) | |
| def test_cache_checks_inputs_sources_and_bytes(sources, cases, tmp_path): | |
| source, _ = sources | |
| cache = tmp_path / "cache" | |
| cache_case(cache, source, cases[0]) | |
| arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source) | |
| assert metadata["complete"] and arrays["mlx"].shape == (1, 1, 1, 3) | |
| changed = (cases[0][0], cases[0][1], 2, {}) | |
| with pytest.raises(ValueError, match="signature"): | |
| validation.load_reference_cache(cache, "5M", changed, source) | |
| model_source = source / "timestral_mlx" / "__init__.py" | |
| model_source.write_text("# changed source\n") | |
| with pytest.raises(ValueError, match="provenance"): | |
| validation.load_reference_cache(cache, "5M", cases[0], source) | |
| model_source.write_text("# reference source\n") | |
| array_file = cache / "5M" / f"{cases[0][0]}.npz" | |
| contents = bytearray(array_file.read_bytes()) | |
| contents[-1] ^= 1 | |
| array_file.write_bytes(contents) | |
| with pytest.raises(ValueError, match="integrity"): | |
| validation.load_reference_cache(cache, "5M", cases[0], source) | |
| def test_incomplete_cache_is_rejected(sources, cases, tmp_path): | |
| source, _ = sources | |
| cache = tmp_path / "cache" | |
| cache_case(cache, source, cases[0], expected=["mlx", "upstream"]) | |
| with pytest.raises(ValueError, match="Incomplete"): | |
| validation.load_reference_cache(cache, "5M", cases[0], source) | |
| def test_numerical_failure_checkpoints_and_stops_by_default( | |
| monkeypatch, sources, cases, tmp_path, keep_going, expected_count): | |
| source, bundles = sources | |
| cache, reports = tmp_path / "cache", tmp_path / "reports" | |
| for case in cases: | |
| cache_case(cache, source, case) | |
| monkeypatch.setattr(validation, "select_cases", lambda *args: cases) | |
| calls = [] | |
| class Candidate: | |
| model = types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="cpu", backend="macos-coreai")) | |
| def forecast(self, *args, **kwargs): | |
| checkpoint = json.loads((reports / "5M.json").read_text()) | |
| assert checkpoint["active_stage"] == "coreai_inference" | |
| calls.append("forecast") | |
| return types.SimpleNamespace(quantiles=np.full((1, 1, 1, 3), 2, np.float32)) | |
| def close(self): | |
| calls.append("close") | |
| monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: Candidate()) | |
| original_import = builtins.__import__ | |
| def no_reference_runtime(name, *args, **kwargs): | |
| if name == "torch" or name.startswith("torch.") or name == "mlx" or name.startswith("mlx."): | |
| raise AssertionError("cached validation imported a reference runtime") | |
| return original_import(name, *args, **kwargs) | |
| monkeypatch.setattr(builtins, "__import__", no_reference_runtime) | |
| assert not validation.validate("5M", source, bundles, reports, selected=[c[0] for c in cases], | |
| keep_going=keep_going, compute_unit="cpu", reference_cache=cache) | |
| report = json.loads((reports / "5M.json").read_text()) | |
| assert len(report["fixtures"]) == expected_count | |
| assert calls == ["forecast"] * expected_count + ["close"] | |
| assert report["status"] == "failed" and not report["passed"] | |
| assert report["environment"]["backend"] == "macos-coreai" | |
| def test_subset_pass_is_not_full_suite_acceptance(monkeypatch, sources, cases, tmp_path): | |
| source, bundles = sources | |
| cache, reports = tmp_path / "cache", tmp_path / "reports" | |
| cache_case(cache, source, cases[0], arrays={ | |
| "mlx": np.ones((1, 1, 1, 3), np.float32), | |
| "upstream": np.ones((1, 1, 1, 3), np.float32), | |
| }) | |
| monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) | |
| candidate = types.SimpleNamespace( | |
| model=types.SimpleNamespace(runtime=types.SimpleNamespace(compute_unit="default", backend="macos-coreai")), | |
| forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32)), | |
| close=lambda: None, | |
| ) | |
| monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate) | |
| assert validation.validate("5M", source, bundles, reports, | |
| selected=[cases[0][0]], reference_cache=cache) | |
| report = json.loads((reports / "5M.json").read_text()) | |
| assert report["passed"] and report["expected_fixture_count"] == 1 | |
| assert report["validation_scope"] == "selected_fixtures" | |
| assert not report["standard_suite_passed"] and not report["extended_suite_passed"] | |
| assert report["fixtures"][0]["upstream_comparison"]["passed"] | |
| def test_interruption_persists_incomplete_report(monkeypatch, sources, cases, tmp_path): | |
| source, bundles = sources | |
| cache, reports = tmp_path / "cache", tmp_path / "reports" | |
| cache_case(cache, source, cases[0]) | |
| monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) | |
| closed = [] | |
| def forecast(*args, **kwargs): | |
| raise validation.ValidationInterrupted(signal.SIGTERM) | |
| candidate = types.SimpleNamespace( | |
| model=types.SimpleNamespace(runtime=types.SimpleNamespace()), forecast=forecast, | |
| close=lambda: closed.append(True), | |
| ) | |
| monkeypatch.setattr(TimestralForecaster, "from_pretrained", lambda *args, **kwargs: candidate) | |
| with pytest.raises(validation.ValidationInterrupted): | |
| validation.validate("5M", source, bundles, reports, selected=[cases[0][0]], reference_cache=cache) | |
| report = json.loads((reports / "5M.json").read_text()) | |
| assert report["status"] == "interrupted" and not report["passed"] | |
| assert report["completed_fixture_count"] == 0 | |
| assert report["interrupted_fixture"] == cases[0][0] | |
| assert closed == [True] | |
| def test_reference_preparation_releases_mlx_before_torch(monkeypatch, sources, cases, tmp_path): | |
| source, _ = sources | |
| upstream = tmp_path / "upstream" | |
| (upstream / "timestral").mkdir(parents=True) | |
| (upstream / "timestral" / "__init__.py").write_text("# original source\n") | |
| cache = tmp_path / "cache" | |
| monkeypatch.setattr(validation, "select_cases", lambda *args: cases[:1]) | |
| refs = [] | |
| class MLX: | |
| def forecast(self, *args, **kwargs): | |
| return types.SimpleNamespace(quantiles=np.ones((1, 1, 1, 3), np.float32)) | |
| def load_mlx(*args): | |
| model = MLX() | |
| refs.append(weakref.ref(model)) | |
| return model | |
| mlx = types.ModuleType("mlx") | |
| mlx.core = types.ModuleType("mlx.core") | |
| mlx.core.clear_cache = lambda: None | |
| reference_module = types.ModuleType("timestral_mlx") | |
| reference_module.TimestralForecaster = types.SimpleNamespace(from_pretrained=load_mlx) | |
| monkeypatch.setitem(sys.modules, "mlx", mlx) | |
| monkeypatch.setitem(sys.modules, "mlx.core", mlx.core) | |
| monkeypatch.setitem(sys.modules, "timestral_mlx", reference_module) | |
| def load_original(*args): | |
| assert refs[0]() is None, "MLX model was still alive when Torch model loaded" | |
| quantiles = types.SimpleNamespace(numpy=lambda: np.ones((1, 1, 1, 3), np.float32)) | |
| return types.SimpleNamespace(forecast=lambda *a, **kw: types.SimpleNamespace(quantiles=quantiles)) | |
| monkeypatch.setattr(validation, "upstream_forecaster", load_original) | |
| validation.prepare_reference_cache("5M", source, cache, upstream, selected=[cases[0][0]]) | |
| arrays, metadata = validation.load_reference_cache(cache, "5M", cases[0], source, upstream) | |
| assert set(arrays) == {"mlx", "upstream"} and metadata["complete"] | |
| assert json.loads((cache / "5M" / "prepare-status.json").read_text())["status"] == "completed" | |