maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
5.89 kB
import copy
import hashlib
import json
import numpy as np
import pytest
import torch
from fastapi.testclient import TestClient
from tfm.api.server import PredictionEngine, PredictRequest, create_app
from tfm.data import FIELD_INDEX, FIELD_SPECS, SCHEMAS, Normalization
from tfm.models import build_model
def request_data():
names = SCHEMAS["thermal"]
values = {"indoor_temp": 22.0, "mass_temp": 22.0, "energy_kwh": 1.0}
return {
"schema": {
"family": "thermal",
"fields": [
{"name": f.name, "unit": f.unit, "role": f.role} for f in FIELD_SPECS if f.name in names
],
},
"state": values,
"history": [{"timestamp_minutes": 0, "state": values}],
"actions": {"idle": {"hvac_power": [0.0] * 8}, "heat": {"hvac_power": [5.0] * 8}},
"exogenous": {"outdoor_temp": [10.0] * 8, "solar_gain": [0.0] * 8, "occupancy_gain": [0.2] * 8},
"horizons_minutes": [15, 120],
"predict": ["indoor_temp"],
}
@pytest.fixture
def engine(tmp_path):
model = build_model("mlp", {"d_model": 16, "ff_dim": 32, "layers": 1, "heads": 4})
checkpoint = tmp_path / "checkpoint.pt"
Normalization(np.zeros(len(FIELD_SPECS)), np.ones(len(FIELD_SPECS)), ["train"]).save(
tmp_path / "normalization.json"
)
torch.save(
{
"normalization_sha256": hashlib.sha256(
(tmp_path / "normalization.json").read_bytes()
).hexdigest(),
"model_kind": "mlp",
"model_config": model.config,
"model_state": model.state_dict(),
"training_system_ids": ["train"],
"validation_selection_system_ids": ["select"],
"calibration_system_ids": ["cal"],
"test_system_ids": ["test"],
},
checkpoint,
)
Normalization(np.zeros(len(FIELD_SPECS)), np.ones(len(FIELD_SPECS)), ["train"]).save(
tmp_path / "normalization.json"
)
return PredictionEngine([checkpoint])
def test_api_batches_alternative_actions_one_pass_and_missing_context(engine):
calls = []
hook = engine.models[0].register_forward_hook(
lambda module, args, output: calls.append(args[0]["current"].shape[0])
)
client = TestClient(create_app(engine))
result = client.post("/v1/predict", json=request_data())
hook.remove()
assert result.status_code == 200, result.text
payload = result.json()
assert calls == [2]
assert payload["indicators"]["insufficient_context"]
assert payload["predictions"]["idle"]["15"]["indoor_temp"]["interval_90"] is None
assert payload["predictions"]["idle"]["15"]["indoor_temp"]["interval_status"] == "unavailable"
@pytest.mark.parametrize(
"mutation",
[
lambda r: r["schema"]["fields"][0].update(unit="F"),
lambda r: r["actions"]["heat"].update(hvac_power=[16] * 8),
lambda r: r["actions"]["heat"].update(hvac_power=["5"] * 8),
lambda r: r["actions"]["heat"].update(hvac_power=[True] * 8),
lambda r: r.update(horizons_minutes=[17]),
lambda r: r.update(horizons_minutes=[15.0]),
lambda r: r["actions"]["heat"].update(hvac_power=[5] * 7),
lambda r: r["state"].update(indoor_temp=None),
lambda r: r.update(hidden_parameters={"insulation": 1}),
lambda r: r["history"][0].update(timestamp_minutes=-15),
],
)
def test_reject_invalid_requests(engine, mutation):
req = copy.deepcopy(request_data())
mutation(req)
response = TestClient(create_app(engine)).post("/v1/predict", json=req)
assert response.status_code == 422, response.text
def test_calibration_bound_to_checkpoint_and_disjoint_systems(engine, tmp_path):
checkpoint = tmp_path / "checkpoint.pt"
# fixture and test share tmp_path
norm = tmp_path / "normalization.json"
calibration = {
"version": "tfm-calibration-v1",
"nominal_coverage": 0.9,
"normalization_version": 1,
"checkpoint_sha256": [hashlib.sha256(checkpoint.read_bytes()).hexdigest()],
"normalization_sha256": hashlib.sha256(norm.read_bytes()).hexdigest(),
"calibration_system_ids": ["cal"],
"entries": {f"{FIELD_INDEX['indoor_temp']}:15": {"multiplier": 1.5, "count": 50}},
}
path = tmp_path / "calibration.json"
path.write_text(json.dumps(calibration))
calibrated = PredictionEngine([checkpoint], path)
output = calibrated.predict(PredictRequest.model_validate(request_data()))
assert output.predictions["idle"]["15"]["indoor_temp"].interval_status == "calibrated"
assert output.predictions["idle"]["120"]["indoor_temp"].interval_status == "unavailable"
calibration["calibration_system_ids"] = ["train"]
path.write_text(json.dumps(calibration))
with pytest.raises(ValueError, match="leakage"):
PredictionEngine([checkpoint], path)
calibration["calibration_system_ids"] = ["cal"]
calibration["checkpoint_sha256"] = ["wrong"]
path.write_text(json.dumps(calibration))
with pytest.raises(ValueError, match="checkpoint"):
PredictionEngine([checkpoint], path)
def test_history_feature_alignment(engine):
req = request_data()
before = {
"timestamp_minutes": -15,
"state": req["state"],
"actions": {"hvac_power": 3.0},
"exogenous": {"outdoor_temp": 10.0, "solar_gain": 0.0, "occupancy_gain": 0.2},
}
req["history"].insert(0, before)
batch, _ = engine.prepare(PredictRequest.model_validate(req))
i = FIELD_INDEX["hvac_power"]
assert batch["history"][0, -2, i] == 3
assert batch["history_mask"][0, -1, i] == 0
assert batch["future_actions"][1, 0, i] == 5
def test_unconfigured_service_does_not_generate_fake_predictions():
response = TestClient(create_app()).post("/v1/predict", json=request_data())
assert response.status_code == 503