import json from types import SimpleNamespace import numpy as np import pytest import torch from tfm.data.schema import FIELD_INDEX, NUM_FIELDS, ROLE_IDS from tfm.evaluation.baselines import HistoryLinearBaseline, PersistenceBaseline, catboost_features from tfm.evaluation.calibration import ( ensemble_moments, finite_quantile, fit_calibration, validate_calibration, ) from tfm.evaluation.counterfactual import action_effect_metrics, branch_rollouts from tfm.evaluation.experiments import ExperimentConfig, run_experiments, stress_observations from tfm.evaluation.metrics import metrics_by_group def test_metric_alignment_masks_units_and_horizons(): scale = np.ones(NUM_FIELDS) scale[0], scale[1] = 2, 10 result = metrics_by_group( np.zeros((2, 3)), np.zeros((2, 3)), np.array([[1.0, 2.0, 999], [3.0, 4.0, 999]]), np.array([[0, 1, 0], [0, 1, 0]]), np.array([[15, 30, 0], [15, 30, 0]]), np.array([[1, 1, 0], [1, 1, 0]]), ["thermal", "battery"], SimpleNamespace(scale=scale), ) assert len(result) == 4 thermal = {r["field_id"]: r for r in result if r["family"] == "thermal"} assert thermal[0]["mae"] == 2 assert thermal[1]["mae"] == 20 assert thermal[0]["horizon_minutes"] == 15 assert thermal[1]["horizon_minutes"] == 30 assert thermal[1]["physical_gaussian_nll"] == pytest.approx( thermal[1]["standardized_gaussian_nll"] + np.log(10) ) assert all(r["count"] == 1 for r in result) def test_finite_sample_calibration_and_leakage_guards(): assert finite_quantile(np.arange(9), 0.9) == 8 assert np.isinf(finite_quantile(np.arange(8), 0.9)) values = np.arange(9, dtype=float).reshape(-1, 1) kwargs = { "calibration_system_ids": ["cal"], "training_system_ids": ["train"], "validation_selection_system_ids": ["val"], "test_system_ids": ["test"], "checkpoint_sha256": ["model"], "normalization_version": 1, "normalization_sha256": "norm", } artifact = fit_calibration( values * 0, values * 0 + 2, values, values * 0, values * 0 + 15, values * 0 + 1, **kwargs ) assert artifact["entries"]["0:15"] == {"multiplier": 4.0, "count": 9} validate_calibration( artifact, checkpoint_sha256=["model"], normalization_version=1, normalization_sha256="norm", forbidden_system_ids=["test"], ) with pytest.raises(ValueError, match="checkpoint"): validate_calibration(artifact, checkpoint_sha256=["different"], normalization_version=1) with pytest.raises(ValueError, match="fingerprint"): validate_calibration( artifact, checkpoint_sha256=["model"], normalization_version=1, normalization_sha256="other" ) kwargs["calibration_system_ids"] = ["test"] with pytest.raises(ValueError, match="disjoint"): fit_calibration(values, values + 1, values, values * 0, values + 15, values + 1, **kwargs) def test_total_ensemble_variance_includes_member_disagreement(): mean, std = ensemble_moments([[-1.0, 4.0], [1.0, 4.0]], [[2.0, 3.0], [2.0, 3.0]]) np.testing.assert_allclose(mean, [0, 4]) np.testing.assert_allclose(std, [np.sqrt(5), 3]) def linear_batch(): h, k = 40, 8 state, action = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"] history = np.zeros((h, NUM_FIELDS), dtype=np.float32) rng = np.random.default_rng(22) controls = rng.normal(size=h) history[:, action] = controls history[0, state] = 1 for i in range(h - 1): history[i + 1, state] = 0.8 * history[i, state] + 0.3 * controls[i] mask = np.zeros_like(history) mask[:, [state, action]] = 1 mask[-1, action] = 0 history[-1, action] = 0 actions = np.zeros((k, NUM_FIELDS), dtype=np.float32) actions[:, action] = rng.normal(size=k) active = np.zeros(NUM_FIELDS, dtype=bool) active[[state, action]] = True batch = { "history": history[None], "history_mask": mask[None], "current": history[None, -1], "current_mask": mask[None, -1], "future_actions": actions[None], "future_exogenous": np.zeros_like(actions)[None], "field_mask": active[None], "query_fields": np.array([[state] * 4]), "query_horizons": np.array([[15.0, 30.0, 60.0, 120.0]]), "role_ids": ROLE_IDS[None], "target": np.full((1, 4), 9999), "target_mask": np.ones((1, 4)), } return {key: torch.tensor(value) for key, value in batch.items()} def test_history_linear_fits_actual_history_and_rolls_planned_controls_only(): batch = linear_batch() model = HistoryLinearBaseline(ROLE_IDS, ridge=1e-8) mean, _ = model(batch) state, control = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"] current = batch["current"][0, state].item() rolled, value = [], current for action in batch["future_actions"][0, :, control]: value = 0.8 * value + 0.3 * action.item() rolled.append(value - current) np.testing.assert_allclose(mean[0], np.array(rolled)[[0, 1, 3, 7]], rtol=1e-5, atol=1e-5) batch["target"].fill_(-9999) np.testing.assert_array_equal(model(batch)[0], mean) assert torch.count_nonzero(PersistenceBaseline()(batch)[0]) == 0 def test_counterfactual_shared_noise_snapshot_and_caller_restoration(): from tfm.simulators import make_simulator sim = make_simulator("battery", 12) snapshot = sim.snapshot() actions = np.zeros((2, 8, NUM_FIELDS)) external = np.zeros((8, NUM_FIELDS)) external[:, FIELD_INDEX["ambient_temp"]] = 20 truth = branch_rollouts(sim, snapshot, actions, external) np.testing.assert_array_equal(truth[0], truth[1]) np.testing.assert_array_equal(sim.state, snapshot["state"]) assert sim.snapshot()["rng_state"] == snapshot["rng_state"] with pytest.raises(ValueError, match="length"): branch_rollouts(sim, snapshot, actions[:, :-1], external) def test_action_effect_ranking_regret_use_same_reference_and_declared_costs(): truth = np.array([[[0.0]], [[2.0]], [[-1.0]]]) predicted = np.array([[[10.0]], [[12.0]], [[9.0]]]) result = action_effect_metrics( predicted, truth, [2, 0, 1], [0, 2, 1], field_names=["temperature"], horizons=[15] ) assert result["effects"][0]["action_effect_mae"] == 0 assert result["regret"] == 2 assert result["ranking_accuracy"] == 0 def test_sensor_failure_corrupts_observations_only(): batch = linear_batch() batch["field_ids"] = torch.arange(NUM_FIELDS)[None] config = ExperimentConfig("fail", "C", sensor_failure_probability=1) stressed = stress_observations(batch, config) assert stressed["history_mask"][0, :, 0].sum() == 0 assert stressed["history"][0, :, 0].sum() == 0 np.testing.assert_array_equal(stressed["target"], batch["target"]) np.testing.assert_array_equal(stressed["future_actions"], batch["future_actions"]) assert batch["history_mask"][0, :, 0].sum() > 0 def test_catboost_query_features_ignore_post_horizon_controls(): batch = linear_batch() batch["history_dt"] = torch.arange(-39, 1)[None] * 15 batch["future_dt"] = torch.arange(1, 9)[None] * 15 source = {k: v.numpy() for k, v in batch.items()} features = catboost_features(source) source["future_actions"][0, 1:, FIELD_INDEX["hvac_power"]] += 999 source["future_exogenous"][0, 1:, FIELD_INDEX["outdoor_temp"]] += 999 modified = catboost_features(source) np.testing.assert_array_equal(features[0], modified[0]) assert not np.array_equal(features[-1], modified[-1]) def test_stress_metrics_decode_biased_anchor_and_reject_absent_current(): from tfm.evaluation.benchmark import collect_predictions source = linear_batch() class OneWindow(torch.utils.data.Dataset): def __len__(self): return 1 def __getitem__(self, _): return {key: value[0] for key, value in source.items()} mean, _, arrays = collect_predictions( PersistenceBaseline(), OneWindow(), "cpu", stress=ExperimentConfig("bias", "C", sensor_bias_std=0.5), seed=22, ) assert np.any(mean != 0) assert np.all(mean == mean[:, :1]) assert arrays["target_mask"].sum() == 4 _, _, arrays = collect_predictions( PersistenceBaseline(), OneWindow(), "cpu", stress=ExperimentConfig("drop", "C", sensor_failure_probability=1), seed=22, ) assert arrays["target_mask"].sum() == 0 def test_context_views_share_h96_eligible_anchors_and_identical_targets(): from tfm.data import Normalization, generate_episode from tfm.evaluation.context import shared_anchor_datasets train = generate_episode("thermal", "train", days=1.25, seed=31, split="train") test = generate_episode("thermal", "test", days=1.25, seed=33, split="test") normalization = Normalization.fit([train]) views = shared_anchor_datasets( [test], {"h24": normalization, "h96": normalization}, {"h24": 24, "h96": 96}, samples=16, seed=2026 ) assert views["h24"].indices == views["h96"].indices assert all(anchor >= 95 for _, anchor in views["h24"].indices) for index in range(16): short, long = views["h24"][index], views["h96"][index] np.testing.assert_array_equal(short["history"], long["history"][-24:]) np.testing.assert_array_equal(short["history_mask"], long["history_mask"][-24:]) for key in ( "current", "current_mask", "query_fields", "query_horizons", "target", "target_mask", "future_actions", "future_exogenous", "future_dt", ): np.testing.assert_array_equal(short[key], long[key]) def test_generalization_reports_absent_artifacts_and_missing_exclusion_honestly(tmp_path): configs = [ ExperimentConfig("main", "B"), ExperimentConfig( "leave_battery", "E", data=str(tmp_path), checkpoint=str(tmp_path / "checkpoint.pt"), excluded_family="battery", ), ] results = run_experiments( configs, lambda _: pytest.fail("Must not execute without provenance"), tmp_path / "experiments.json" ) assert all(r["status"] == "unrun" for r in results) assert "provenance" in results[1]["reason"] assert json.loads((tmp_path / "experiments.json").read_text())["experiments"] == results def test_benchmark_end_to_end_on_exact_heldout_windows(tmp_path, monkeypatch): from tfm.data import Normalization, generate_dataset from tfm.evaluation.benchmark import benchmark, parser, sha256 from tfm.models import build_model from tfm.training.train import load_manifest_episodes, split_validation_systems data, run, output = tmp_path / "data", tmp_path / "run", tmp_path / "report" generate_dataset(data, systems=9, days=0.5, seed=10) _, splits = load_manifest_episodes(data) selection, calibration = split_validation_systems(splits["validation"], 42) run.mkdir() Normalization.fit(splits["train"]).save(run / "normalization.json") model = build_model("gru", {"history_steps": 8, "d_model": 16, "heads": 4, "layers": 1, "ff_dim": 32}) def ids(episodes): return sorted({e.system_id for e in episodes}) torch.save( { "model_kind": "gru", "model_config": model.config, "model_state": model.state_dict(), "normalization_sha256": sha256(run / "normalization.json"), "training_system_ids": ids(splits["train"]), "validation_selection_system_ids": ids(selection), "calibration_system_ids": ids(calibration), "test_system_ids": ids(splits["test"]), }, run / "checkpoint.pt", ) monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False) args = parser().parse_args( [ "--data", str(data), "--checkpoint", str(run / "checkpoint.pt"), "--output", str(output), "--samples", "6", "--calibration-samples", "12", "--device", "cpu", "--counterfactual-samples", "2", "--latency-repetitions", "2", ] ) result = benchmark(args) assert result["models"]["foundation_model"]["status"] == "completed" assert result["models"]["history_linear_system_id"]["counterfactual"]["samples"] == 2 assert result["external_datasets"]["status"] == "unrun" assert not set(result["test_system_ids"]) & set(result["calibration_system_ids"]) assert result["inference"][0]["p95_ms"] > 0 assert (output / "benchmark.json").exists() assert (output / "calibration.json").exists() assert result["manifest_sha256"] == sha256(data / "manifest.json") assert sum(result["window_sampling"]["family_window_counts"].values()) == 6 assert len(result["window_sampling"]["anchor_sha256"]) == 64 primary = result["models"]["foundation_model"] calibrated = sum( row["count"] for row in primary["metrics"] if row["interval_kind"] == "heldout_calibrated" ) observed = sum(row["count"] for row in primary["metrics"]) assert primary["calibrated_query_fraction"] == calibrated / observed if calibrated < observed: assert primary["calibration_status"] == "heldout_partial"