Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tests/test_models.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 9.99 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tests/test_models.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tests/test_models.py
-
curl -L -o test_models.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tests/test_models.py
9.99 kB
| import copy | |
| from types import SimpleNamespace | |
| import pytest | |
| import torch | |
| from tfm.data.schema import FIELD_SPECS, ROLE_IDS, UNIT_IDS | |
| from tfm.models import build_model, gaussian_nll | |
| from tfm.training.train import load_checkpoint, split_validation_systems, state_dict_sha256 | |
| def batch_fixture(batch_size=2): | |
| torch.manual_seed(15) | |
| fields = len(FIELD_SPECS) | |
| states = [i for i, f in enumerate(FIELD_SPECS) if f.role == "state"] | |
| history, future = 4, 8 | |
| active = torch.ones(batch_size, fields, dtype=torch.bool) | |
| active[:, -1] = False | |
| mask = active[:, None].expand(-1, history, -1).clone() | |
| result = { | |
| "history": torch.randn(batch_size, history, fields), | |
| "history_mask": mask, | |
| "history_dt": torch.arange(-(history - 1) * 15, 1, 15).float().expand(batch_size, -1), | |
| "current": torch.randn(batch_size, fields), | |
| "current_mask": active.clone(), | |
| "future_actions": torch.randn(batch_size, future, fields), | |
| "future_exogenous": torch.randn(batch_size, future, fields), | |
| "future_dt": torch.arange(15, 121, 15).float().expand(batch_size, -1), | |
| "field_mask": active, | |
| "field_ids": torch.arange(fields).expand(batch_size, -1), | |
| "unit_ids": torch.tensor(UNIT_IDS.copy()).expand(batch_size, -1), | |
| "role_ids": torch.tensor(ROLE_IDS.copy()).expand(batch_size, -1), | |
| "query_fields": torch.tensor([states[0], states[1], states[0], fields - 1]).expand(batch_size, -1), | |
| "query_horizons": torch.tensor([15.0, 30.0, 120.0, 120.0]).expand(batch_size, -1), | |
| "target": torch.randn(batch_size, 4), | |
| "target_mask": torch.tensor([True, True, True, False]).expand(batch_size, -1), | |
| } | |
| return result | |
| def small_model(kind): | |
| return build_model(kind, {"history_steps": 4, "d_model": 32, "layers": 2, "heads": 4, "ff_dim": 64}) | |
| def test_shape_action_sensitivity_and_finite_gradients(kind): | |
| batch = batch_fixture() | |
| model = small_model(kind) | |
| mean, log_std = model(batch) | |
| assert mean.shape == log_std.shape == (2, 4) | |
| assert torch.isfinite(mean).all() and torch.isfinite(log_std).all() | |
| changed = copy.deepcopy(batch) | |
| action = next(i for i, field in enumerate(FIELD_SPECS) if field.role == "action") | |
| changed["future_actions"][:, :, action] += 10 | |
| changed_mean, _ = model(changed) | |
| assert (changed_mean[:, :3] - mean[:, :3]).abs().max() > 1e-5 | |
| loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) | |
| loss.backward() | |
| assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()) | |
| assert model.encoder.numeric[0].weight.grad.abs().sum() > 0 | |
| def test_field_permutation_preserves_semantic_queries(kind): | |
| batch = batch_fixture() | |
| permutation = torch.randperm(len(FIELD_SPECS)) | |
| shuffled = copy.deepcopy(batch) | |
| for key in [ | |
| "history", | |
| "history_mask", | |
| "current", | |
| "current_mask", | |
| "future_actions", | |
| "future_exogenous", | |
| "field_mask", | |
| "field_ids", | |
| "unit_ids", | |
| "role_ids", | |
| ]: | |
| shuffled[key] = batch[key][..., permutation] | |
| model = small_model(kind).eval() | |
| with torch.no_grad(): | |
| before = model(batch) | |
| after = model(shuffled) | |
| for expected, actual in zip(before, after): | |
| torch.testing.assert_close(expected, actual, atol=2e-6, rtol=1e-5) | |
| def test_padded_and_unobserved_values_do_not_leak(kind): | |
| batch = batch_fixture() | |
| batch["history_mask"][:, 0, 0] = False | |
| batch["current_mask"][:, 1] = False | |
| changed = copy.deepcopy(batch) | |
| for key in ["history", "current", "future_actions", "future_exogenous"]: | |
| changed[key][..., -1] = float("nan") | |
| changed["history"][:, 0, 0] = float("nan") | |
| changed["current"][:, 1] = float("nan") | |
| model = small_model(kind).eval() | |
| with torch.no_grad(): | |
| before, after = model(batch), model(changed) | |
| for expected, actual in zip(before, after): | |
| torch.testing.assert_close(expected, actual) | |
| assert (actual[:, -1] == 0).all() | |
| def test_later_actions_cannot_affect_earlier_horizon(kind): | |
| batch = batch_fixture() | |
| changed = copy.deepcopy(batch) | |
| changed["future_actions"][:, -1] += 20 | |
| changed["future_exogenous"][:, -1] += 20 | |
| model = small_model(kind).eval() | |
| with torch.no_grad(): | |
| before, after = model(batch), model(changed) | |
| for expected, actual in zip(before, after): | |
| torch.testing.assert_close(expected[:, :2], actual[:, :2], atol=2e-6, rtol=1e-5) | |
| def test_missing_targets_have_zero_gradient_and_finite_loss(): | |
| mean = torch.tensor([[0.2, float("nan")]], requires_grad=True) | |
| log_std = torch.tensor([[0.1, float("nan")]], requires_grad=True) | |
| target = torch.tensor([[0.8, float("nan")]]) | |
| loss = gaussian_nll(mean, log_std, target, torch.tensor([[True, False]])) | |
| assert torch.isfinite(loss) | |
| loss.backward() | |
| assert mean.grad[0, 1] == log_std.grad[0, 1] == 0 | |
| empty = gaussian_nll(mean, log_std, target, torch.zeros_like(target)) | |
| assert empty == 0 and torch.isfinite(empty) | |
| def test_default_transformer_capacity(): | |
| model = build_model("transformer") | |
| assert model.config["d_model"] == 128 | |
| assert len(model.blocks) == 4 | |
| assert model.config["heads"] == 4 and model.config["ff_dim"] == 512 | |
| assert 1_000_000 <= sum(p.numel() for p in model.parameters()) <= 3_000_000 | |
| def test_history_ablation_ignores_older_history_and_metadata(): | |
| batch = batch_fixture() | |
| changed = copy.deepcopy(batch) | |
| changed["history"][:, :-1] += 100 | |
| changed["hidden_parameters"] = {"capacity": 99} | |
| model = build_model( | |
| "transformer", {"d_model": 32, "layers": 1, "heads": 4, "ff_dim": 64, "use_history": False} | |
| ) | |
| with torch.no_grad(): | |
| before, after = model(batch), model(changed) | |
| for expected, actual in zip(before, after): | |
| torch.testing.assert_close(expected, actual) | |
| def test_masked_input_gradient_is_zero(): | |
| batch = batch_fixture() | |
| batch["history"].requires_grad_() | |
| batch["future_actions"].requires_grad_() | |
| batch["history_mask"][:, 0, 0] = False | |
| model = small_model("transformer") | |
| mean, log_std = model(batch) | |
| gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]).backward() | |
| assert (batch["history"].grad[:, 0, 0] == 0).all() | |
| assert (batch["history"].grad[:, :, -1] == 0).all() | |
| assert (batch["future_actions"].grad[:, :, -1] == 0).all() | |
| def test_checkpoint_roundtrip(tmp_path): | |
| model = small_model("transformer").eval() | |
| path = tmp_path / "checkpoint.pt" | |
| torch.save( | |
| { | |
| "model_kind": "transformer", | |
| "model_config": model.config, | |
| "model_state": model.state_dict(), | |
| "seed": 3, | |
| "epoch": 1, | |
| }, | |
| path, | |
| ) | |
| restored, metadata = load_checkpoint(path) | |
| assert metadata["seed"] == 3 | |
| batch = batch_fixture() | |
| with torch.no_grad(): | |
| before, after = model(batch), restored(batch) | |
| for expected, actual in zip(before, after): | |
| torch.testing.assert_close(expected, actual) | |
| def test_validation_calibration_are_system_disjoint_and_reproducible(): | |
| episodes = [ | |
| SimpleNamespace(system_id=f"{family}{i}", family_id=family) | |
| for family in ["thermal", "tank", "battery"] | |
| for i in range(4) | |
| for _ in range(2) | |
| ] | |
| selection, calibration = split_validation_systems(episodes, 23) | |
| def ids(items): | |
| return {e.system_id for e in items} | |
| assert ids(selection).isdisjoint(ids(calibration)) | |
| assert ids(selection) | ids(calibration) == ids(episodes) | |
| assert ids(calibration) == ids(split_validation_systems(episodes, 23)[1]) | |
| assert split_validation_systems(episodes[:1], 23)[1] == [] | |
| def test_default_full_history_supports_dynamic_query_count(): | |
| batch = batch_fixture(batch_size=1) | |
| # Exercise the actual default H24 model rather than only small fixtures. | |
| for key in ("history", "history_mask"): | |
| batch[key] = batch[key].repeat(1, 6, 1) | |
| batch["history_dt"] = torch.arange(-23 * 15, 1, 15).float()[None] | |
| model = build_model("transformer").eval() | |
| with torch.inference_mode(): | |
| full = model(batch) | |
| for count in (1, 2, 7): | |
| dynamic = dict(batch) | |
| indices = torch.arange(count) % batch["query_fields"].shape[1] | |
| for key in ("query_fields", "query_horizons"): | |
| dynamic[key] = batch[key][:, indices] | |
| actual = model(dynamic) | |
| for expected, observed in zip(full, actual): | |
| assert observed.shape == (1, count) | |
| torch.testing.assert_close(observed, expected[:, indices], atol=2e-6, rtol=1e-5) | |
| def test_checkpoint_rejects_corrupted_artifact_metadata(tmp_path, corruption): | |
| model = small_model("transformer") | |
| state = model.state_dict() | |
| metadata = { | |
| "checkpoint_version": "tfm-checkpoint-v1", | |
| "schema_version": "tfm-schema-v1", | |
| "field_names": [field.name for field in FIELD_SPECS], | |
| "model_kind": "transformer", | |
| "model_config": model.config, | |
| "model_state": state, | |
| "model_state_sha256": state_dict_sha256(state), | |
| } | |
| if corruption == "weights": | |
| state["output.3.bias"][0] += 1 | |
| message = "digest" | |
| elif corruption == "vocabulary": | |
| metadata["field_names"].reverse() | |
| message = "vocabulary" | |
| else: | |
| metadata["checkpoint_version"] = "tfm-checkpoint-v2" | |
| message = "version" | |
| path = tmp_path / "checkpoint.pt" | |
| torch.save(metadata, path) | |
| with pytest.raises(ValueError, match=message): | |
| load_checkpoint(path) | |