import copy from types import SimpleNamespace import pytest import torch from tfm.data.schema import FIELD_SPECS, ROLE_IDS, UNIT_IDS from tfm.models import build_model, gaussian_nll from tfm.training.train import load_checkpoint, split_validation_systems, state_dict_sha256 def batch_fixture(batch_size=2): torch.manual_seed(15) fields = len(FIELD_SPECS) states = [i for i, f in enumerate(FIELD_SPECS) if f.role == "state"] history, future = 4, 8 active = torch.ones(batch_size, fields, dtype=torch.bool) active[:, -1] = False mask = active[:, None].expand(-1, history, -1).clone() result = { "history": torch.randn(batch_size, history, fields), "history_mask": mask, "history_dt": torch.arange(-(history - 1) * 15, 1, 15).float().expand(batch_size, -1), "current": torch.randn(batch_size, fields), "current_mask": active.clone(), "future_actions": torch.randn(batch_size, future, fields), "future_exogenous": torch.randn(batch_size, future, fields), "future_dt": torch.arange(15, 121, 15).float().expand(batch_size, -1), "field_mask": active, "field_ids": torch.arange(fields).expand(batch_size, -1), "unit_ids": torch.tensor(UNIT_IDS.copy()).expand(batch_size, -1), "role_ids": torch.tensor(ROLE_IDS.copy()).expand(batch_size, -1), "query_fields": torch.tensor([states[0], states[1], states[0], fields - 1]).expand(batch_size, -1), "query_horizons": torch.tensor([15.0, 30.0, 120.0, 120.0]).expand(batch_size, -1), "target": torch.randn(batch_size, 4), "target_mask": torch.tensor([True, True, True, False]).expand(batch_size, -1), } return result def small_model(kind): return build_model(kind, {"history_steps": 4, "d_model": 32, "layers": 2, "heads": 4, "ff_dim": 64}) @pytest.mark.parametrize("kind", ["transformer", "mlp", "gru"]) def test_shape_action_sensitivity_and_finite_gradients(kind): batch = batch_fixture() model = small_model(kind) mean, log_std = model(batch) assert mean.shape == log_std.shape == (2, 4) assert torch.isfinite(mean).all() and torch.isfinite(log_std).all() changed = copy.deepcopy(batch) action = next(i for i, field in enumerate(FIELD_SPECS) if field.role == "action") changed["future_actions"][:, :, action] += 10 changed_mean, _ = model(changed) assert (changed_mean[:, :3] - mean[:, :3]).abs().max() > 1e-5 loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) loss.backward() assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()) assert model.encoder.numeric[0].weight.grad.abs().sum() > 0 @pytest.mark.parametrize("kind", ["transformer", "mlp", "gru"]) def test_field_permutation_preserves_semantic_queries(kind): batch = batch_fixture() permutation = torch.randperm(len(FIELD_SPECS)) shuffled = copy.deepcopy(batch) for key in [ "history", "history_mask", "current", "current_mask", "future_actions", "future_exogenous", "field_mask", "field_ids", "unit_ids", "role_ids", ]: shuffled[key] = batch[key][..., permutation] model = small_model(kind).eval() with torch.no_grad(): before = model(batch) after = model(shuffled) for expected, actual in zip(before, after): torch.testing.assert_close(expected, actual, atol=2e-6, rtol=1e-5) @pytest.mark.parametrize("kind", ["transformer", "mlp", "gru"]) def test_padded_and_unobserved_values_do_not_leak(kind): batch = batch_fixture() batch["history_mask"][:, 0, 0] = False batch["current_mask"][:, 1] = False changed = copy.deepcopy(batch) for key in ["history", "current", "future_actions", "future_exogenous"]: changed[key][..., -1] = float("nan") changed["history"][:, 0, 0] = float("nan") changed["current"][:, 1] = float("nan") model = small_model(kind).eval() with torch.no_grad(): before, after = model(batch), model(changed) for expected, actual in zip(before, after): torch.testing.assert_close(expected, actual) assert (actual[:, -1] == 0).all() @pytest.mark.parametrize("kind", ["transformer", "mlp", "gru"]) def test_later_actions_cannot_affect_earlier_horizon(kind): batch = batch_fixture() changed = copy.deepcopy(batch) changed["future_actions"][:, -1] += 20 changed["future_exogenous"][:, -1] += 20 model = small_model(kind).eval() with torch.no_grad(): before, after = model(batch), model(changed) for expected, actual in zip(before, after): torch.testing.assert_close(expected[:, :2], actual[:, :2], atol=2e-6, rtol=1e-5) def test_missing_targets_have_zero_gradient_and_finite_loss(): mean = torch.tensor([[0.2, float("nan")]], requires_grad=True) log_std = torch.tensor([[0.1, float("nan")]], requires_grad=True) target = torch.tensor([[0.8, float("nan")]]) loss = gaussian_nll(mean, log_std, target, torch.tensor([[True, False]])) assert torch.isfinite(loss) loss.backward() assert mean.grad[0, 1] == log_std.grad[0, 1] == 0 empty = gaussian_nll(mean, log_std, target, torch.zeros_like(target)) assert empty == 0 and torch.isfinite(empty) def test_default_transformer_capacity(): model = build_model("transformer") assert model.config["d_model"] == 128 assert len(model.blocks) == 4 assert model.config["heads"] == 4 and model.config["ff_dim"] == 512 assert 1_000_000 <= sum(p.numel() for p in model.parameters()) <= 3_000_000 def test_history_ablation_ignores_older_history_and_metadata(): batch = batch_fixture() changed = copy.deepcopy(batch) changed["history"][:, :-1] += 100 changed["hidden_parameters"] = {"capacity": 99} model = build_model( "transformer", {"d_model": 32, "layers": 1, "heads": 4, "ff_dim": 64, "use_history": False} ) with torch.no_grad(): before, after = model(batch), model(changed) for expected, actual in zip(before, after): torch.testing.assert_close(expected, actual) def test_masked_input_gradient_is_zero(): batch = batch_fixture() batch["history"].requires_grad_() batch["future_actions"].requires_grad_() batch["history_mask"][:, 0, 0] = False model = small_model("transformer") mean, log_std = model(batch) gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]).backward() assert (batch["history"].grad[:, 0, 0] == 0).all() assert (batch["history"].grad[:, :, -1] == 0).all() assert (batch["future_actions"].grad[:, :, -1] == 0).all() def test_checkpoint_roundtrip(tmp_path): model = small_model("transformer").eval() path = tmp_path / "checkpoint.pt" torch.save( { "model_kind": "transformer", "model_config": model.config, "model_state": model.state_dict(), "seed": 3, "epoch": 1, }, path, ) restored, metadata = load_checkpoint(path) assert metadata["seed"] == 3 batch = batch_fixture() with torch.no_grad(): before, after = model(batch), restored(batch) for expected, actual in zip(before, after): torch.testing.assert_close(expected, actual) def test_validation_calibration_are_system_disjoint_and_reproducible(): episodes = [ SimpleNamespace(system_id=f"{family}{i}", family_id=family) for family in ["thermal", "tank", "battery"] for i in range(4) for _ in range(2) ] selection, calibration = split_validation_systems(episodes, 23) def ids(items): return {e.system_id for e in items} assert ids(selection).isdisjoint(ids(calibration)) assert ids(selection) | ids(calibration) == ids(episodes) assert ids(calibration) == ids(split_validation_systems(episodes, 23)[1]) assert split_validation_systems(episodes[:1], 23)[1] == [] def test_default_full_history_supports_dynamic_query_count(): batch = batch_fixture(batch_size=1) # Exercise the actual default H24 model rather than only small fixtures. for key in ("history", "history_mask"): batch[key] = batch[key].repeat(1, 6, 1) batch["history_dt"] = torch.arange(-23 * 15, 1, 15).float()[None] model = build_model("transformer").eval() with torch.inference_mode(): full = model(batch) for count in (1, 2, 7): dynamic = dict(batch) indices = torch.arange(count) % batch["query_fields"].shape[1] for key in ("query_fields", "query_horizons"): dynamic[key] = batch[key][:, indices] actual = model(dynamic) for expected, observed in zip(full, actual): assert observed.shape == (1, count) torch.testing.assert_close(observed, expected[:, indices], atol=2e-6, rtol=1e-5) @pytest.mark.parametrize("corruption", ["weights", "vocabulary", "version"]) def test_checkpoint_rejects_corrupted_artifact_metadata(tmp_path, corruption): model = small_model("transformer") state = model.state_dict() metadata = { "checkpoint_version": "tfm-checkpoint-v1", "schema_version": "tfm-schema-v1", "field_names": [field.name for field in FIELD_SPECS], "model_kind": "transformer", "model_config": model.config, "model_state": state, "model_state_sha256": state_dict_sha256(state), } if corruption == "weights": state["output.3.bias"][0] += 1 message = "digest" elif corruption == "vocabulary": metadata["field_names"].reverse() message = "vocabulary" else: metadata["checkpoint_version"] = "tfm-checkpoint-v2" message = "version" path = tmp_path / "checkpoint.pt" torch.save(metadata, path) with pytest.raises(ValueError, match=message): load_checkpoint(path)