File size: 5,885 Bytes
1baf215 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | import copy
import hashlib
import json
import numpy as np
import pytest
import torch
from fastapi.testclient import TestClient
from tfm.api.server import PredictionEngine, PredictRequest, create_app
from tfm.data import FIELD_INDEX, FIELD_SPECS, SCHEMAS, Normalization
from tfm.models import build_model
def request_data():
names = SCHEMAS["thermal"]
values = {"indoor_temp": 22.0, "mass_temp": 22.0, "energy_kwh": 1.0}
return {
"schema": {
"family": "thermal",
"fields": [
{"name": f.name, "unit": f.unit, "role": f.role} for f in FIELD_SPECS if f.name in names
],
},
"state": values,
"history": [{"timestamp_minutes": 0, "state": values}],
"actions": {"idle": {"hvac_power": [0.0] * 8}, "heat": {"hvac_power": [5.0] * 8}},
"exogenous": {"outdoor_temp": [10.0] * 8, "solar_gain": [0.0] * 8, "occupancy_gain": [0.2] * 8},
"horizons_minutes": [15, 120],
"predict": ["indoor_temp"],
}
@pytest.fixture
def engine(tmp_path):
model = build_model("mlp", {"d_model": 16, "ff_dim": 32, "layers": 1, "heads": 4})
checkpoint = tmp_path / "checkpoint.pt"
Normalization(np.zeros(len(FIELD_SPECS)), np.ones(len(FIELD_SPECS)), ["train"]).save(
tmp_path / "normalization.json"
)
torch.save(
{
"normalization_sha256": hashlib.sha256(
(tmp_path / "normalization.json").read_bytes()
).hexdigest(),
"model_kind": "mlp",
"model_config": model.config,
"model_state": model.state_dict(),
"training_system_ids": ["train"],
"validation_selection_system_ids": ["select"],
"calibration_system_ids": ["cal"],
"test_system_ids": ["test"],
},
checkpoint,
)
Normalization(np.zeros(len(FIELD_SPECS)), np.ones(len(FIELD_SPECS)), ["train"]).save(
tmp_path / "normalization.json"
)
return PredictionEngine([checkpoint])
def test_api_batches_alternative_actions_one_pass_and_missing_context(engine):
calls = []
hook = engine.models[0].register_forward_hook(
lambda module, args, output: calls.append(args[0]["current"].shape[0])
)
client = TestClient(create_app(engine))
result = client.post("/v1/predict", json=request_data())
hook.remove()
assert result.status_code == 200, result.text
payload = result.json()
assert calls == [2]
assert payload["indicators"]["insufficient_context"]
assert payload["predictions"]["idle"]["15"]["indoor_temp"]["interval_90"] is None
assert payload["predictions"]["idle"]["15"]["indoor_temp"]["interval_status"] == "unavailable"
@pytest.mark.parametrize(
"mutation",
[
lambda r: r["schema"]["fields"][0].update(unit="F"),
lambda r: r["actions"]["heat"].update(hvac_power=[16] * 8),
lambda r: r["actions"]["heat"].update(hvac_power=["5"] * 8),
lambda r: r["actions"]["heat"].update(hvac_power=[True] * 8),
lambda r: r.update(horizons_minutes=[17]),
lambda r: r.update(horizons_minutes=[15.0]),
lambda r: r["actions"]["heat"].update(hvac_power=[5] * 7),
lambda r: r["state"].update(indoor_temp=None),
lambda r: r.update(hidden_parameters={"insulation": 1}),
lambda r: r["history"][0].update(timestamp_minutes=-15),
],
)
def test_reject_invalid_requests(engine, mutation):
req = copy.deepcopy(request_data())
mutation(req)
response = TestClient(create_app(engine)).post("/v1/predict", json=req)
assert response.status_code == 422, response.text
def test_calibration_bound_to_checkpoint_and_disjoint_systems(engine, tmp_path):
checkpoint = tmp_path / "checkpoint.pt"
# fixture and test share tmp_path
norm = tmp_path / "normalization.json"
calibration = {
"version": "tfm-calibration-v1",
"nominal_coverage": 0.9,
"normalization_version": 1,
"checkpoint_sha256": [hashlib.sha256(checkpoint.read_bytes()).hexdigest()],
"normalization_sha256": hashlib.sha256(norm.read_bytes()).hexdigest(),
"calibration_system_ids": ["cal"],
"entries": {f"{FIELD_INDEX['indoor_temp']}:15": {"multiplier": 1.5, "count": 50}},
}
path = tmp_path / "calibration.json"
path.write_text(json.dumps(calibration))
calibrated = PredictionEngine([checkpoint], path)
output = calibrated.predict(PredictRequest.model_validate(request_data()))
assert output.predictions["idle"]["15"]["indoor_temp"].interval_status == "calibrated"
assert output.predictions["idle"]["120"]["indoor_temp"].interval_status == "unavailable"
calibration["calibration_system_ids"] = ["train"]
path.write_text(json.dumps(calibration))
with pytest.raises(ValueError, match="leakage"):
PredictionEngine([checkpoint], path)
calibration["calibration_system_ids"] = ["cal"]
calibration["checkpoint_sha256"] = ["wrong"]
path.write_text(json.dumps(calibration))
with pytest.raises(ValueError, match="checkpoint"):
PredictionEngine([checkpoint], path)
def test_history_feature_alignment(engine):
req = request_data()
before = {
"timestamp_minutes": -15,
"state": req["state"],
"actions": {"hvac_power": 3.0},
"exogenous": {"outdoor_temp": 10.0, "solar_gain": 0.0, "occupancy_gain": 0.2},
}
req["history"].insert(0, before)
batch, _ = engine.prepare(PredictRequest.model_validate(req))
i = FIELD_INDEX["hvac_power"]
assert batch["history"][0, -2, i] == 3
assert batch["history_mask"][0, -1, i] == 0
assert batch["future_actions"][1, 0, i] == 5
def test_unconfigured_service_does_not_generate_fake_predictions():
response = TestClient(create_app()).post("/v1/predict", json=request_data())
assert response.status_code == 503
|