File size: 13,589 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
import json
from types import SimpleNamespace

import numpy as np
import pytest
import torch

from tfm.data.schema import FIELD_INDEX, NUM_FIELDS, ROLE_IDS
from tfm.evaluation.baselines import HistoryLinearBaseline, PersistenceBaseline, catboost_features
from tfm.evaluation.calibration import (
    ensemble_moments,
    finite_quantile,
    fit_calibration,
    validate_calibration,
)
from tfm.evaluation.counterfactual import action_effect_metrics, branch_rollouts
from tfm.evaluation.experiments import ExperimentConfig, run_experiments, stress_observations
from tfm.evaluation.metrics import metrics_by_group


def test_metric_alignment_masks_units_and_horizons():
    scale = np.ones(NUM_FIELDS)
    scale[0], scale[1] = 2, 10
    result = metrics_by_group(
        np.zeros((2, 3)),
        np.zeros((2, 3)),
        np.array([[1.0, 2.0, 999], [3.0, 4.0, 999]]),
        np.array([[0, 1, 0], [0, 1, 0]]),
        np.array([[15, 30, 0], [15, 30, 0]]),
        np.array([[1, 1, 0], [1, 1, 0]]),
        ["thermal", "battery"],
        SimpleNamespace(scale=scale),
    )
    assert len(result) == 4
    thermal = {r["field_id"]: r for r in result if r["family"] == "thermal"}
    assert thermal[0]["mae"] == 2
    assert thermal[1]["mae"] == 20
    assert thermal[0]["horizon_minutes"] == 15
    assert thermal[1]["horizon_minutes"] == 30
    assert thermal[1]["physical_gaussian_nll"] == pytest.approx(
        thermal[1]["standardized_gaussian_nll"] + np.log(10)
    )
    assert all(r["count"] == 1 for r in result)


def test_finite_sample_calibration_and_leakage_guards():
    assert finite_quantile(np.arange(9), 0.9) == 8
    assert np.isinf(finite_quantile(np.arange(8), 0.9))
    values = np.arange(9, dtype=float).reshape(-1, 1)
    kwargs = {
        "calibration_system_ids": ["cal"],
        "training_system_ids": ["train"],
        "validation_selection_system_ids": ["val"],
        "test_system_ids": ["test"],
        "checkpoint_sha256": ["model"],
        "normalization_version": 1,
        "normalization_sha256": "norm",
    }
    artifact = fit_calibration(
        values * 0, values * 0 + 2, values, values * 0, values * 0 + 15, values * 0 + 1, **kwargs
    )
    assert artifact["entries"]["0:15"] == {"multiplier": 4.0, "count": 9}
    validate_calibration(
        artifact,
        checkpoint_sha256=["model"],
        normalization_version=1,
        normalization_sha256="norm",
        forbidden_system_ids=["test"],
    )
    with pytest.raises(ValueError, match="checkpoint"):
        validate_calibration(artifact, checkpoint_sha256=["different"], normalization_version=1)
    with pytest.raises(ValueError, match="fingerprint"):
        validate_calibration(
            artifact, checkpoint_sha256=["model"], normalization_version=1, normalization_sha256="other"
        )
    kwargs["calibration_system_ids"] = ["test"]
    with pytest.raises(ValueError, match="disjoint"):
        fit_calibration(values, values + 1, values, values * 0, values + 15, values + 1, **kwargs)


def test_total_ensemble_variance_includes_member_disagreement():
    mean, std = ensemble_moments([[-1.0, 4.0], [1.0, 4.0]], [[2.0, 3.0], [2.0, 3.0]])
    np.testing.assert_allclose(mean, [0, 4])
    np.testing.assert_allclose(std, [np.sqrt(5), 3])


def linear_batch():
    h, k = 40, 8
    state, action = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"]
    history = np.zeros((h, NUM_FIELDS), dtype=np.float32)
    rng = np.random.default_rng(22)
    controls = rng.normal(size=h)
    history[:, action] = controls
    history[0, state] = 1
    for i in range(h - 1):
        history[i + 1, state] = 0.8 * history[i, state] + 0.3 * controls[i]
    mask = np.zeros_like(history)
    mask[:, [state, action]] = 1
    mask[-1, action] = 0
    history[-1, action] = 0
    actions = np.zeros((k, NUM_FIELDS), dtype=np.float32)
    actions[:, action] = rng.normal(size=k)
    active = np.zeros(NUM_FIELDS, dtype=bool)
    active[[state, action]] = True
    batch = {
        "history": history[None],
        "history_mask": mask[None],
        "current": history[None, -1],
        "current_mask": mask[None, -1],
        "future_actions": actions[None],
        "future_exogenous": np.zeros_like(actions)[None],
        "field_mask": active[None],
        "query_fields": np.array([[state] * 4]),
        "query_horizons": np.array([[15.0, 30.0, 60.0, 120.0]]),
        "role_ids": ROLE_IDS[None],
        "target": np.full((1, 4), 9999),
        "target_mask": np.ones((1, 4)),
    }
    return {key: torch.tensor(value) for key, value in batch.items()}


def test_history_linear_fits_actual_history_and_rolls_planned_controls_only():
    batch = linear_batch()
    model = HistoryLinearBaseline(ROLE_IDS, ridge=1e-8)
    mean, _ = model(batch)
    state, control = FIELD_INDEX["indoor_temp"], FIELD_INDEX["hvac_power"]
    current = batch["current"][0, state].item()
    rolled, value = [], current
    for action in batch["future_actions"][0, :, control]:
        value = 0.8 * value + 0.3 * action.item()
        rolled.append(value - current)
    np.testing.assert_allclose(mean[0], np.array(rolled)[[0, 1, 3, 7]], rtol=1e-5, atol=1e-5)
    batch["target"].fill_(-9999)
    np.testing.assert_array_equal(model(batch)[0], mean)
    assert torch.count_nonzero(PersistenceBaseline()(batch)[0]) == 0


def test_counterfactual_shared_noise_snapshot_and_caller_restoration():
    from tfm.simulators import make_simulator

    sim = make_simulator("battery", 12)
    snapshot = sim.snapshot()
    actions = np.zeros((2, 8, NUM_FIELDS))
    external = np.zeros((8, NUM_FIELDS))
    external[:, FIELD_INDEX["ambient_temp"]] = 20
    truth = branch_rollouts(sim, snapshot, actions, external)
    np.testing.assert_array_equal(truth[0], truth[1])
    np.testing.assert_array_equal(sim.state, snapshot["state"])
    assert sim.snapshot()["rng_state"] == snapshot["rng_state"]
    with pytest.raises(ValueError, match="length"):
        branch_rollouts(sim, snapshot, actions[:, :-1], external)


def test_action_effect_ranking_regret_use_same_reference_and_declared_costs():
    truth = np.array([[[0.0]], [[2.0]], [[-1.0]]])
    predicted = np.array([[[10.0]], [[12.0]], [[9.0]]])
    result = action_effect_metrics(
        predicted, truth, [2, 0, 1], [0, 2, 1], field_names=["temperature"], horizons=[15]
    )
    assert result["effects"][0]["action_effect_mae"] == 0
    assert result["regret"] == 2
    assert result["ranking_accuracy"] == 0


def test_sensor_failure_corrupts_observations_only():
    batch = linear_batch()
    batch["field_ids"] = torch.arange(NUM_FIELDS)[None]
    config = ExperimentConfig("fail", "C", sensor_failure_probability=1)
    stressed = stress_observations(batch, config)
    assert stressed["history_mask"][0, :, 0].sum() == 0
    assert stressed["history"][0, :, 0].sum() == 0
    np.testing.assert_array_equal(stressed["target"], batch["target"])
    np.testing.assert_array_equal(stressed["future_actions"], batch["future_actions"])
    assert batch["history_mask"][0, :, 0].sum() > 0


def test_catboost_query_features_ignore_post_horizon_controls():
    batch = linear_batch()
    batch["history_dt"] = torch.arange(-39, 1)[None] * 15
    batch["future_dt"] = torch.arange(1, 9)[None] * 15
    source = {k: v.numpy() for k, v in batch.items()}
    features = catboost_features(source)
    source["future_actions"][0, 1:, FIELD_INDEX["hvac_power"]] += 999
    source["future_exogenous"][0, 1:, FIELD_INDEX["outdoor_temp"]] += 999
    modified = catboost_features(source)
    np.testing.assert_array_equal(features[0], modified[0])
    assert not np.array_equal(features[-1], modified[-1])


def test_stress_metrics_decode_biased_anchor_and_reject_absent_current():
    from tfm.evaluation.benchmark import collect_predictions

    source = linear_batch()

    class OneWindow(torch.utils.data.Dataset):
        def __len__(self):
            return 1

        def __getitem__(self, _):
            return {key: value[0] for key, value in source.items()}

    mean, _, arrays = collect_predictions(
        PersistenceBaseline(),
        OneWindow(),
        "cpu",
        stress=ExperimentConfig("bias", "C", sensor_bias_std=0.5),
        seed=22,
    )
    assert np.any(mean != 0)
    assert np.all(mean == mean[:, :1])
    assert arrays["target_mask"].sum() == 4
    _, _, arrays = collect_predictions(
        PersistenceBaseline(),
        OneWindow(),
        "cpu",
        stress=ExperimentConfig("drop", "C", sensor_failure_probability=1),
        seed=22,
    )
    assert arrays["target_mask"].sum() == 0


def test_context_views_share_h96_eligible_anchors_and_identical_targets():
    from tfm.data import Normalization, generate_episode
    from tfm.evaluation.context import shared_anchor_datasets

    train = generate_episode("thermal", "train", days=1.25, seed=31, split="train")
    test = generate_episode("thermal", "test", days=1.25, seed=33, split="test")
    normalization = Normalization.fit([train])
    views = shared_anchor_datasets(
        [test], {"h24": normalization, "h96": normalization}, {"h24": 24, "h96": 96}, samples=16, seed=2026
    )
    assert views["h24"].indices == views["h96"].indices
    assert all(anchor >= 95 for _, anchor in views["h24"].indices)
    for index in range(16):
        short, long = views["h24"][index], views["h96"][index]
        np.testing.assert_array_equal(short["history"], long["history"][-24:])
        np.testing.assert_array_equal(short["history_mask"], long["history_mask"][-24:])
        for key in (
            "current",
            "current_mask",
            "query_fields",
            "query_horizons",
            "target",
            "target_mask",
            "future_actions",
            "future_exogenous",
            "future_dt",
        ):
            np.testing.assert_array_equal(short[key], long[key])


def test_generalization_reports_absent_artifacts_and_missing_exclusion_honestly(tmp_path):
    configs = [
        ExperimentConfig("main", "B"),
        ExperimentConfig(
            "leave_battery",
            "E",
            data=str(tmp_path),
            checkpoint=str(tmp_path / "checkpoint.pt"),
            excluded_family="battery",
        ),
    ]
    results = run_experiments(
        configs, lambda _: pytest.fail("Must not execute without provenance"), tmp_path / "experiments.json"
    )
    assert all(r["status"] == "unrun" for r in results)
    assert "provenance" in results[1]["reason"]
    assert json.loads((tmp_path / "experiments.json").read_text())["experiments"] == results


def test_benchmark_end_to_end_on_exact_heldout_windows(tmp_path, monkeypatch):
    from tfm.data import Normalization, generate_dataset
    from tfm.evaluation.benchmark import benchmark, parser, sha256
    from tfm.models import build_model
    from tfm.training.train import load_manifest_episodes, split_validation_systems

    data, run, output = tmp_path / "data", tmp_path / "run", tmp_path / "report"
    generate_dataset(data, systems=9, days=0.5, seed=10)
    _, splits = load_manifest_episodes(data)
    selection, calibration = split_validation_systems(splits["validation"], 42)
    run.mkdir()
    Normalization.fit(splits["train"]).save(run / "normalization.json")
    model = build_model("gru", {"history_steps": 8, "d_model": 16, "heads": 4, "layers": 1, "ff_dim": 32})

    def ids(episodes):
        return sorted({e.system_id for e in episodes})

    torch.save(
        {
            "model_kind": "gru",
            "model_config": model.config,
            "model_state": model.state_dict(),
            "normalization_sha256": sha256(run / "normalization.json"),
            "training_system_ids": ids(splits["train"]),
            "validation_selection_system_ids": ids(selection),
            "calibration_system_ids": ids(calibration),
            "test_system_ids": ids(splits["test"]),
        },
        run / "checkpoint.pt",
    )
    monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
    args = parser().parse_args(
        [
            "--data",
            str(data),
            "--checkpoint",
            str(run / "checkpoint.pt"),
            "--output",
            str(output),
            "--samples",
            "6",
            "--calibration-samples",
            "12",
            "--device",
            "cpu",
            "--counterfactual-samples",
            "2",
            "--latency-repetitions",
            "2",
        ]
    )
    result = benchmark(args)
    assert result["models"]["foundation_model"]["status"] == "completed"
    assert result["models"]["history_linear_system_id"]["counterfactual"]["samples"] == 2
    assert result["external_datasets"]["status"] == "unrun"
    assert not set(result["test_system_ids"]) & set(result["calibration_system_ids"])
    assert result["inference"][0]["p95_ms"] > 0
    assert (output / "benchmark.json").exists()
    assert (output / "calibration.json").exists()
    assert result["manifest_sha256"] == sha256(data / "manifest.json")
    assert sum(result["window_sampling"]["family_window_counts"].values()) == 6
    assert len(result["window_sampling"]["anchor_sha256"]) == 64
    primary = result["models"]["foundation_model"]
    calibrated = sum(
        row["count"] for row in primary["metrics"] if row["interval_kind"] == "heldout_calibrated"
    )
    observed = sum(row["count"] for row in primary["metrics"])
    assert primary["calibrated_query_fraction"] == calibrated / observed
    if calibrated < observed:
        assert primary["calibration_status"] == "heldout_partial"