Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tests/test_evaluation.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tests/test_evaluation.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tests/test_evaluation.py
-
curl -L -o test_evaluation.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tests/test_evaluation.py
13.6 kB
| import json | |
| from types import SimpleNamespace | |
| import numpy as np | |
| import pytest | |
| import torch | |
| from tfm.data.schema import FIELD_INDEX, NUM_FIELDS, ROLE_IDS | |
| from tfm.evaluation.baselines import HistoryLinearBaseline, PersistenceBaseline, catboost_features | |
| from tfm.evaluation.calibration import ( | |
| ensemble_moments, | |
| finite_quantile, | |
| fit_calibration, | |
| validate_calibration, | |
| ) | |
| from tfm.evaluation.counterfactual import action_effect_metrics, branch_rollouts | |
| from tfm.evaluation.experiments import ExperimentConfig, run_experiments, stress_observations | |
| from tfm.evaluation.metrics import metrics_by_group | |
| def test_metric_alignment_masks_units_and_horizons(): | |
| scale = np.ones(NUM_FIELDS) | |
| scale[0], scale[1] = 2, 10 | |
| result = metrics_by_group( | |
| np.zeros((2, 3)), | |
| np.zeros((2, 3)), | |
| np.array([[1.0, 2.0, 999], [3.0, 4.0, 999]]), | |
| np.array([[0, 1, 0], [0, 1, 0]]), | |
| np.array([[15, 30, 0], [15, 30, 0]]), | |
| np.array([[1, 1, 0], [1, 1, 0]]), | |
| ["thermal", "battery"], | |
| SimpleNamespace(scale=scale), | |
| ) | |
| assert len(result) == 4 | |
| thermal = {r["field_id"]: r for r in result if r["family"] == "thermal"} | |
| assert thermal[0]["mae"] == 2 | |
| assert thermal[1]["mae"] == 20 | |
| assert thermal[0]["horizon_minutes"] == 15 | |
| assert thermal[1]["horizon_minutes"] == 30 | |
| assert thermal[1]["physical_gaussian_nll"] == pytest.approx( | |
| thermal[1]["standardized_gaussian_nll"] + np.log(10) | |
| ) | |
| assert all(r["count"] == 1 for r in result) | |
| def test_finite_sample_calibration_and_leakage_guards(): | |
| assert finite_quantile(np.arange(9), 0.9) == 8 | |
| assert np.isinf(finite_quantile(np.arange(8), 0.9)) | |
| values = np.arange(9, dtype=float).reshape(-1, 1) | |
| kwargs = { | |
| "calibration_system_ids": ["cal"], | |
| "training_system_ids": ["train"], | |
| "validation_selection_system_ids": ["val"], | |
| "test_system_ids": ["test"], | |
| "checkpoint_sha256": ["model"], | |
| "normalization_version": 1, | |
| "normalization_sha256": "norm", | |
| } | |
| artifact = fit_calibration( | |
| values * 0, values * 0 + 2, values, values * 0, values * 0 + 15, values * 0 + 1, **kwargs | |
| ) | |
| assert artifact["entries"]["0:15"] == {"multiplier": 4.0, "count": 9} | |
| validate_calibration( | |
| artifact, | |
| checkpoint_sha256=["model"], | |
| normalization_version=1, | |
| normalization_sha256="norm", | |
| forbidden_system_ids=["test"], | |
| ) | |
| with pytest.raises(ValueError, match="checkpoint"): | |
| validate_calibration(artifact, checkpoint_sha256=["different"], normalization_version=1) | |
| with pytest.raises(ValueError, match="fingerprint"): | |
| validate_calibration( | |
| artifact, checkpoint_sha256=["model"], normalization_version=1, normalization_sha256="other" | |
| ) | |
| kwargs["calibration_system_ids"] = ["test"] | |
| with pytest.raises(ValueError, match="disjoint"): | |
| fit_calibration(values, values + 1, values, values * 0, values + 15, values + 1, **kwargs) | |
| def test_total_ensemble_variance_includes_member_disagreement(): | |
| mean, std = ensemble_moments([[-1.0, 4.0], [1.0, 4.0]], [[2.0, 3.0], [2.0, 3.0]]) | |
| np.testing.assert_allclose(mean, [0, 4]) | |
| np.testing.assert_allclose(std, [np.sqrt(5), 3]) | |
| def linear_batch(): | |
| h, k = 40, 8 | |
| state, action = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"] | |
| history = np.zeros((h, NUM_FIELDS), dtype=np.float32) | |
| rng = np.random.default_rng(22) | |
| controls = rng.normal(size=h) | |
| history[:, action] = controls | |
| history[0, state] = 1 | |
| for i in range(h - 1): | |
| history[i + 1, state] = 0.8 * history[i, state] + 0.3 * controls[i] | |
| mask = np.zeros_like(history) | |
| mask[:, [state, action]] = 1 | |
| mask[-1, action] = 0 | |
| history[-1, action] = 0 | |
| actions = np.zeros((k, NUM_FIELDS), dtype=np.float32) | |
| actions[:, action] = rng.normal(size=k) | |
| active = np.zeros(NUM_FIELDS, dtype=bool) | |
| active[[state, action]] = True | |
| batch = { | |
| "history": history[None], | |
| "history_mask": mask[None], | |
| "current": history[None, -1], | |
| "current_mask": mask[None, -1], | |
| "future_actions": actions[None], | |
| "future_exogenous": np.zeros_like(actions)[None], | |
| "field_mask": active[None], | |
| "query_fields": np.array([[state] * 4]), | |
| "query_horizons": np.array([[15.0, 30.0, 60.0, 120.0]]), | |
| "role_ids": ROLE_IDS[None], | |
| "target": np.full((1, 4), 9999), | |
| "target_mask": np.ones((1, 4)), | |
| } | |
| return {key: torch.tensor(value) for key, value in batch.items()} | |
| def test_history_linear_fits_actual_history_and_rolls_planned_controls_only(): | |
| batch = linear_batch() | |
| model = HistoryLinearBaseline(ROLE_IDS, ridge=1e-8) | |
| mean, _ = model(batch) | |
| state, control = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"] | |
| current = batch["current"][0, state].item() | |
| rolled, value = [], current | |
| for action in batch["future_actions"][0, :, control]: | |
| value = 0.8 * value + 0.3 * action.item() | |
| rolled.append(value - current) | |
| np.testing.assert_allclose(mean[0], np.array(rolled)[[0, 1, 3, 7]], rtol=1e-5, atol=1e-5) | |
| batch["target"].fill_(-9999) | |
| np.testing.assert_array_equal(model(batch)[0], mean) | |
| assert torch.count_nonzero(PersistenceBaseline()(batch)[0]) == 0 | |
| def test_counterfactual_shared_noise_snapshot_and_caller_restoration(): | |
| from tfm.simulators import make_simulator | |
| sim = make_simulator("battery", 12) | |
| snapshot = sim.snapshot() | |
| actions = np.zeros((2, 8, NUM_FIELDS)) | |
| external = np.zeros((8, NUM_FIELDS)) | |
| external[:, FIELD_INDEX["ambient_temp"]] = 20 | |
| truth = branch_rollouts(sim, snapshot, actions, external) | |
| np.testing.assert_array_equal(truth[0], truth[1]) | |
| np.testing.assert_array_equal(sim.state, snapshot["state"]) | |
| assert sim.snapshot()["rng_state"] == snapshot["rng_state"] | |
| with pytest.raises(ValueError, match="length"): | |
| branch_rollouts(sim, snapshot, actions[:, :-1], external) | |
| def test_action_effect_ranking_regret_use_same_reference_and_declared_costs(): | |
| truth = np.array([[[0.0]], [[2.0]], [[-1.0]]]) | |
| predicted = np.array([[[10.0]], [[12.0]], [[9.0]]]) | |
| result = action_effect_metrics( | |
| predicted, truth, [2, 0, 1], [0, 2, 1], field_names=["temperature"], horizons=[15] | |
| ) | |
| assert result["effects"][0]["action_effect_mae"] == 0 | |
| assert result["regret"] == 2 | |
| assert result["ranking_accuracy"] == 0 | |
| def test_sensor_failure_corrupts_observations_only(): | |
| batch = linear_batch() | |
| batch["field_ids"] = torch.arange(NUM_FIELDS)[None] | |
| config = ExperimentConfig("fail", "C", sensor_failure_probability=1) | |
| stressed = stress_observations(batch, config) | |
| assert stressed["history_mask"][0, :, 0].sum() == 0 | |
| assert stressed["history"][0, :, 0].sum() == 0 | |
| np.testing.assert_array_equal(stressed["target"], batch["target"]) | |
| np.testing.assert_array_equal(stressed["future_actions"], batch["future_actions"]) | |
| assert batch["history_mask"][0, :, 0].sum() > 0 | |
| def test_catboost_query_features_ignore_post_horizon_controls(): | |
| batch = linear_batch() | |
| batch["history_dt"] = torch.arange(-39, 1)[None] * 15 | |
| batch["future_dt"] = torch.arange(1, 9)[None] * 15 | |
| source = {k: v.numpy() for k, v in batch.items()} | |
| features = catboost_features(source) | |
| source["future_actions"][0, 1:, FIELD_INDEX["hvac_power"]] += 999 | |
| source["future_exogenous"][0, 1:, FIELD_INDEX["outdoor_temp"]] += 999 | |
| modified = catboost_features(source) | |
| np.testing.assert_array_equal(features[0], modified[0]) | |
| assert not np.array_equal(features[-1], modified[-1]) | |
| def test_stress_metrics_decode_biased_anchor_and_reject_absent_current(): | |
| from tfm.evaluation.benchmark import collect_predictions | |
| source = linear_batch() | |
| class OneWindow(torch.utils.data.Dataset): | |
| def __len__(self): | |
| return 1 | |
| def __getitem__(self, _): | |
| return {key: value[0] for key, value in source.items()} | |
| mean, _, arrays = collect_predictions( | |
| PersistenceBaseline(), | |
| OneWindow(), | |
| "cpu", | |
| stress=ExperimentConfig("bias", "C", sensor_bias_std=0.5), | |
| seed=22, | |
| ) | |
| assert np.any(mean != 0) | |
| assert np.all(mean == mean[:, :1]) | |
| assert arrays["target_mask"].sum() == 4 | |
| _, _, arrays = collect_predictions( | |
| PersistenceBaseline(), | |
| OneWindow(), | |
| "cpu", | |
| stress=ExperimentConfig("drop", "C", sensor_failure_probability=1), | |
| seed=22, | |
| ) | |
| assert arrays["target_mask"].sum() == 0 | |
| def test_context_views_share_h96_eligible_anchors_and_identical_targets(): | |
| from tfm.data import Normalization, generate_episode | |
| from tfm.evaluation.context import shared_anchor_datasets | |
| train = generate_episode("thermal", "train", days=1.25, seed=31, split="train") | |
| test = generate_episode("thermal", "test", days=1.25, seed=33, split="test") | |
| normalization = Normalization.fit([train]) | |
| views = shared_anchor_datasets( | |
| [test], {"h24": normalization, "h96": normalization}, {"h24": 24, "h96": 96}, samples=16, seed=2026 | |
| ) | |
| assert views["h24"].indices == views["h96"].indices | |
| assert all(anchor >= 95 for _, anchor in views["h24"].indices) | |
| for index in range(16): | |
| short, long = views["h24"][index], views["h96"][index] | |
| np.testing.assert_array_equal(short["history"], long["history"][-24:]) | |
| np.testing.assert_array_equal(short["history_mask"], long["history_mask"][-24:]) | |
| for key in ( | |
| "current", | |
| "current_mask", | |
| "query_fields", | |
| "query_horizons", | |
| "target", | |
| "target_mask", | |
| "future_actions", | |
| "future_exogenous", | |
| "future_dt", | |
| ): | |
| np.testing.assert_array_equal(short[key], long[key]) | |
| def test_generalization_reports_absent_artifacts_and_missing_exclusion_honestly(tmp_path): | |
| configs = [ | |
| ExperimentConfig("main", "B"), | |
| ExperimentConfig( | |
| "leave_battery", | |
| "E", | |
| data=str(tmp_path), | |
| checkpoint=str(tmp_path / "checkpoint.pt"), | |
| excluded_family="battery", | |
| ), | |
| ] | |
| results = run_experiments( | |
| configs, lambda _: pytest.fail("Must not execute without provenance"), tmp_path / "experiments.json" | |
| ) | |
| assert all(r["status"] == "unrun" for r in results) | |
| assert "provenance" in results[1]["reason"] | |
| assert json.loads((tmp_path / "experiments.json").read_text())["experiments"] == results | |
| def test_benchmark_end_to_end_on_exact_heldout_windows(tmp_path, monkeypatch): | |
| from tfm.data import Normalization, generate_dataset | |
| from tfm.evaluation.benchmark import benchmark, parser, sha256 | |
| from tfm.models import build_model | |
| from tfm.training.train import load_manifest_episodes, split_validation_systems | |
| data, run, output = tmp_path / "data", tmp_path / "run", tmp_path / "report" | |
| generate_dataset(data, systems=9, days=0.5, seed=10) | |
| _, splits = load_manifest_episodes(data) | |
| selection, calibration = split_validation_systems(splits["validation"], 42) | |
| run.mkdir() | |
| Normalization.fit(splits["train"]).save(run / "normalization.json") | |
| model = build_model("gru", {"history_steps": 8, "d_model": 16, "heads": 4, "layers": 1, "ff_dim": 32}) | |
| def ids(episodes): | |
| return sorted({e.system_id for e in episodes}) | |
| torch.save( | |
| { | |
| "model_kind": "gru", | |
| "model_config": model.config, | |
| "model_state": model.state_dict(), | |
| "normalization_sha256": sha256(run / "normalization.json"), | |
| "training_system_ids": ids(splits["train"]), | |
| "validation_selection_system_ids": ids(selection), | |
| "calibration_system_ids": ids(calibration), | |
| "test_system_ids": ids(splits["test"]), | |
| }, | |
| run / "checkpoint.pt", | |
| ) | |
| monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False) | |
| args = parser().parse_args( | |
| [ | |
| "--data", | |
| str(data), | |
| "--checkpoint", | |
| str(run / "checkpoint.pt"), | |
| "--output", | |
| str(output), | |
| "--samples", | |
| "6", | |
| "--calibration-samples", | |
| "12", | |
| "--device", | |
| "cpu", | |
| "--counterfactual-samples", | |
| "2", | |
| "--latency-repetitions", | |
| "2", | |
| ] | |
| ) | |
| result = benchmark(args) | |
| assert result["models"]["foundation_model"]["status"] == "completed" | |
| assert result["models"]["history_linear_system_id"]["counterfactual"]["samples"] == 2 | |
| assert result["external_datasets"]["status"] == "unrun" | |
| assert not set(result["test_system_ids"]) & set(result["calibration_system_ids"]) | |
| assert result["inference"][0]["p95_ms"] > 0 | |
| assert (output / "benchmark.json").exists() | |
| assert (output / "calibration.json").exists() | |
| assert result["manifest_sha256"] == sha256(data / "manifest.json") | |
| assert sum(result["window_sampling"]["family_window_counts"].values()) == 6 | |
| assert len(result["window_sampling"]["anchor_sha256"]) == 64 | |
| primary = result["models"]["foundation_model"] | |
| calibrated = sum( | |
| row["count"] for row in primary["metrics"] if row["interval_kind"] == "heldout_calibrated" | |
| ) | |
| observed = sum(row["count"] for row in primary["metrics"]) | |
| assert primary["calibrated_query_fraction"] == calibrated / observed | |
| if calibrated < observed: | |
| assert primary["calibration_status"] == "heldout_partial" | |