import json import numpy as np import pytest import torch from tfm.data import ( FIELD_INDEX, FIELD_SPECS, ROLE_IDS, Normalization, WindowDataset, generate_dataset, generate_episode, load_episode, load_manifest_episodes, save_episode, ) def test_episode_roundtrip_repeatability(tmp_path): a = generate_episode("thermal", "a", days=1, seed=10, split="train") b = generate_episode("thermal", "a", days=1, seed=10, split="train") np.testing.assert_array_equal(a.values, b.values) np.testing.assert_array_equal(a.observation_mask, b.observation_mask) save_episode(a, tmp_path / "episode.npz") restored = load_episode(tmp_path / "episode.npz") assert restored.hidden_parameters == a.hidden_parameters assert restored.schema == a.schema and restored.seed == a.seed np.testing.assert_array_equal(restored.actions, a.actions) np.testing.assert_array_equal(restored.values, a.values) def test_stratified_system_partitions_and_training_normalization(tmp_path): manifest = generate_dataset(tmp_path, systems=30, days=0.5, seed=7) ids = { split: {e["system_id"] for e in manifest["episodes"] if e["split"] == split} for split in ("train", "validation", "test") } assert not ids["train"] & ids["validation"] and not ids["train"] & ids["test"] assert not ids["validation"] & ids["test"] assert len(set.union(*ids.values())) == 30 for split in ids: assert {e["family_id"] for e in manifest["episodes"] if e["split"] == split} == { "thermal", "tank", "battery", } normalization = Normalization.load(tmp_path / "normalization.json") assert set(normalization.training_system_ids) == ids["train"] expected = Normalization.fit(load_manifest_episodes(tmp_path, "train")) np.testing.assert_array_equal(normalization.mean, expected.mean) with pytest.raises(ValueError, match="training"): Normalization.fit(load_manifest_episodes(tmp_path, "test")) other = generate_dataset(tmp_path / "other", systems=30, days=0.5, seed=7) assert other == manifest def test_window_alignment_masks_and_no_future_state_leakage(): ep = generate_episode("tank", "train", days=1, seed=10, split="train", missing_probability=0) norm = Normalization.fit([ep]) dataset = WindowDataset([ep], norm, history_steps=4, samples=10, seed=3) clone = WindowDataset([ep], norm, history_steps=4, samples=10, seed=3) assert dataset.indices == clone.indices _i, t = dataset.indices[0] batch = dataset[0] np.testing.assert_array_equal(batch["history_dt"], [-45, -30, -15, 0]) np.testing.assert_array_equal(batch["future_dt"], np.arange(1, 9) * 15) assert not batch["current_mask"][ROLE_IDS != 0].any() assert not batch["history"][-1, ROLE_IDS != 0].any() assert not batch["future_actions"][:, ROLE_IDS != 1].any() assert not batch["future_exogenous"][:, ROLE_IDS != 2].any() assert set(batch) == { "history", "history_mask", "history_dt", "current", "current_mask", "future_actions", "future_exogenous", "future_dt", "field_mask", "field_ids", "unit_ids", "role_ids", "query_fields", "query_horizons", "target", "target_mask", } for q in range(16): f = batch["query_fields"][q].item() offset = int(batch["query_horizons"][q].item() / 15) assert batch["target"][q].item() == pytest.approx( (ep.values[t + offset, f] - ep.values[t, f]) / norm.scale[f] ) # Future sensor data changes labels but cannot alter any model input. ep.values[t + 1 :, ROLE_IDS == 0] += 1000 changed = dataset[0] for key in batch: if key not in ("target", "target_mask"): assert torch.equal(batch[key], changed[key]), key assert not torch.equal(batch["target"], changed["target"]) # Hidden capacities are retained for simulator truth only. ep.hidden_parameters["capacity"] = 1e9 assert torch.equal(changed["history"], dataset[0]["history"]) def test_missing_masks_padded_queries_and_training_statistics(): ep = generate_episode("battery", "train", days=1, seed=11, split="train", missing_probability=0) masked = FIELD_INDEX["battery_temp"] ep.observation_mask[:, masked] = False ep.values[:, masked] = 1e9 norm = Normalization.fit([ep]) assert norm.mean[masked] == 0 and norm.scale[masked] == 1 batch = WindowDataset([ep], norm, samples=1)[0] assert not batch["history"][:, masked].any() assert not batch["history_mask"][:, masked].any() assert not batch["target_mask"][4:8].any() assert not batch["target_mask"][12:].any() assert not batch["target"][12:].any() assert batch["history"].dtype == torch.float32 assert batch["field_ids"].dtype == torch.int64 assert batch["target_mask"].dtype == torch.bool # Public schema maximum does not encode instance capacity. assert FIELD_SPECS[FIELD_INDEX["water_level"]].maximum == 50 def test_historical_controls_use_explicit_arrays(): ep = generate_episode("thermal", "train", days=1, seed=12, split="train", missing_probability=0) norm = Normalization.fit([ep]) dataset = WindowDataset([ep], norm, history_steps=4, samples=1) _, t = dataset.indices[0] field = FIELD_INDEX["hvac_power"] ep.actions[t - 1, field] = 7.5 ep.values[t - 1, field] = -999 assert dataset[0]["history"][-2, field].item() == pytest.approx( (7.5 - norm.mean[field]) / norm.scale[field] ) def test_mixed_policy_proportions_and_optional_parquet(tmp_path): ep = generate_episode("battery", "a", days=7, seed=99) fractions = np.bincount(ep.policy_ids, minlength=4) / len(ep.policy_ids) np.testing.assert_allclose(fractions, [0.35, 0.30, 0.25, 0.10], atol=0.05) pytest.importorskip("pyarrow") import pyarrow.parquet as pq from tfm.data import save_episode_parquet save_episode_parquet(ep, tmp_path / "episode.parquet") table = pq.read_table(tmp_path / "episode.parquet") assert table.num_rows == len(ep.values) * len(ep.schema) assert "hidden_parameters" in json.loads(table.schema.metadata[b"tfm_episode"]) def test_masked_nan_is_not_a_training_statistic_or_feature(tmp_path): ep = generate_episode("battery", "train", days=1, seed=14, split="train", missing_probability=0) field = FIELD_INDEX["battery_temp"] ep.values[:, field] = np.nan ep.observation_mask[:, field] = False save_episode(ep, tmp_path / "missing.npz") restored = load_episode(tmp_path / "missing.npz") norm = Normalization.fit([restored]) assert norm.mean[field] == 0 and norm.scale[field] == 1 batch = WindowDataset([restored], norm, samples=1)[0] assert all(torch.isfinite(value).all() for value in batch.values()) ep.observation_mask[0, field] = True with pytest.raises(ValueError, match="Observed"): ep.__post_init__() @pytest.mark.parametrize("systems,expected", [(1200, (840, 180, 180)), (90, (63, 14, 13))]) def test_required_split_counts_without_simulation(systems, expected): from tfm.data.generate import split_systems families = ("thermal", "tank", "battery") assigned = [families[i % 3] for i in range(systems)] splits = split_systems(assigned, seed=42) assert tuple(list(splits.values()).count(name) for name in ("train", "validation", "test")) == expected assert splits == split_systems(assigned, seed=42) for family in families: assert {splits[i] for i, assigned_family in enumerate(assigned) if assigned_family == family} == { "train", "validation", "test", } def test_policy_segments_full_range_holds_structured_ramps_and_feedback(): from tfm.data.generate import POLICY_NAMES, POLICY_WEIGHTS, _rounded_counts from tfm.simulators import make_simulator ep = generate_episode("battery", "policy-audit", days=7, seed=135) assert POLICY_NAMES == ("random_piecewise", "structured_step_ramp", "feedback", "boundary") np.testing.assert_array_equal( np.bincount(ep.policy_ids, minlength=4), _rounded_counts(len(ep.actions), POLICY_WEIGHTS) ) field = FIELD_INDEX["battery_power"] random_values, random_lengths, ramps, structured_steps = [], [], 0, 0 for segment_id in np.unique(ep.policy_segment_ids): selected = np.where(ep.policy_segment_ids == segment_id)[0] assert 2 <= len(selected) <= 16 policy = ep.policy_ids[selected[0]] assert np.all(ep.policy_ids[selected] == policy) controls = ep.actions[selected, field] if policy == 0: np.testing.assert_array_equal(controls, np.repeat(controls[0], len(controls))) random_values.append(controls[0]) random_lengths.append(len(controls)) elif policy == 1: differences = np.diff(controls) if ( len(differences) > 1 and np.allclose(differences, differences[0]) and abs(differences[0]) > 1e-6 ): ramps += 1 elif np.count_nonzero(abs(differences) > 1e-6) == 1: structured_steps += 1 elif policy == 3: assert controls[0] in (-20, 20) np.testing.assert_array_equal(controls, np.repeat(controls[0], len(controls))) assert len(set(random_lengths)) >= 5 assert min(random_values) < -12 and max(random_values) > 12 # full public range, beyond prior 60% limit assert ramps > 0 and structured_steps > 0 sim = make_simulator("battery", ep.seed) for t in range(len(ep.actions)): if ep.policy_ids[t] == 2: assert ep.actions[t, field] == pytest.approx(np.clip((0.55 - sim.state[0]) * 15, -20, 20)) sim.step(ep.actions[t], ep.exogenous[t]) def test_policy_segment_ids_survive_episode_roundtrip(tmp_path): ep = generate_episode("tank", "segment-roundtrip", days=0.5, seed=17) save_episode(ep, tmp_path / "episode.npz") restored = load_episode(tmp_path / "episode.npz") np.testing.assert_array_equal(restored.policy_ids, ep.policy_ids) np.testing.assert_array_equal(restored.policy_segment_ids, ep.policy_segment_ids)