File size: 11,465 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
"""Seeded mixed-policy episode generation and stratified system partitions.

Exogenous values are forecast inputs at transition t, never future state sensors.
Thermal/battery temperatures follow daily sinusoids plus correlated innovations;
solar peaks in daytime, occupancy and tank demand follow daily schedules.
"""

import argparse
import json
from pathlib import Path

import numpy as np

from .episodes import Episode, load_manifest_episodes, save_episode, save_episode_parquet
from .schema import DT_MINUTES, FIELD_INDEX, FIELD_SPECS, GENERATOR_VERSION, NUM_FIELDS, SCHEMAS, role_fields

POLICY_NAMES = ("random_piecewise", "structured_step_ramp", "feedback", "boundary")
POLICY_WEIGHTS = (0.35, 0.30, 0.25, 0.10)


def generate_exogenous(family, steps, seed):
    rng = np.random.default_rng(seed)
    hour = np.arange(steps) * DT_MINUTES / 60 % 24
    result = np.zeros((steps, NUM_FIELDS))
    innovation = np.zeros(steps)
    for t in range(1, steps):
        innovation[t] = 0.94 * innovation[t - 1] + rng.normal(0, 0.25)
    daily = rng.uniform(8, 24) + rng.uniform(3, 9) * np.sin((hour - 9) * np.pi / 12) + innovation
    if family == "thermal":
        result[:, FIELD_INDEX["outdoor_temp"]] = daily
        result[:, FIELD_INDEX["solar_gain"]] = rng.uniform(0.5, 3) * np.maximum(
            0, np.sin((hour - 6) * np.pi / 12)
        )
        result[:, FIELD_INDEX["occupancy_gain"]] = 0.12 + ((hour >= 17) | (hour < 8)) * rng.uniform(0.1, 0.7)
    elif family == "tank":
        result[:, FIELD_INDEX["demand"]] = np.clip(
            0.4
            + 2 * np.exp(-(((hour - 8) / 2) ** 2))
            + 2 * np.exp(-(((hour - 19) / 3) ** 2))
            + rng.normal(0, 0.15, steps),
            0,
            10,
        )
        result[:, FIELD_INDEX["inlet_temp"]] = 13 + 0.15 * daily
    elif family == "battery":
        result[:, FIELD_INDEX["ambient_temp"]] = daily
    else:
        raise ValueError(f"Unknown family {family}")
    return result


def _rounded_counts(size, weights):
    """Largest-remainder integer quotas, preserving the requested total."""
    expected = np.asarray(weights, dtype=float) * size
    counts = np.floor(expected).astype(int)
    order = np.argsort(-(expected - counts), kind="stable")
    counts[order[: size - int(counts.sum())]] += 1
    return counts


def split_systems(assigned_families, seed=0):
    """Deterministic stratified 70/15/15 system split with global integer quotas.

    Each family has at least one system in each partition. At small sizes this
    constraint takes priority over fractional quotas. Equal rounding ties go to
    validation before test, so 90 systems split 63/14/13, and 1200 split 840/180/180.
    """
    assigned_families = list(assigned_families)
    families = tuple(dict.fromkeys(assigned_families))
    if not families:
        raise ValueError("Cannot split an empty system list")
    groups = [np.array([i for i, f in enumerate(assigned_families) if f == family]) for family in families]
    if any(len(ids) < 3 for ids in groups):
        raise ValueError("Stratification requires at least three systems per family")
    weights = (0.70, 0.15, 0.15)
    target = _rounded_counts(len(assigned_families), weights)
    counts = np.array([np.maximum(1, np.floor(len(ids) * np.array(weights)).astype(int)) for ids in groups])
    for row, ids in zip(counts, groups):
        while row.sum() > len(ids):
            eligible = np.where(row > 1, row, -1)
            row[int(np.argmax(eligible))] -= 1
    for row, ids in zip(counts, groups):
        while row.sum() < len(ids):
            deficits = target - counts.sum(axis=0)
            row[int(np.argmax(deficits))] += 1
    rng = np.random.default_rng(seed)
    splits = {}
    for ids, (n_train, n_val, _) in zip(groups, counts):
        rng.shuffle(ids)
        for split, selected in (
            ("train", ids[:n_train]),
            ("validation", ids[n_train : n_train + n_val]),
            ("test", ids[n_train + n_val :]),
        ):
            splits.update({int(i): split for i in selected})
    return splits


def policy_segments(steps, rng):
    """Random 2..16-step segments selected by remaining 35/30/25/10 quotas.

    Segment boundaries remain explicit even for adjacent segments of the same
    policy. Sampled lengths are adjusted at quota tails to avoid one-step holds.
    Only very short episodes whose entire policy quota is one step require a
    one-step truncated segment. Exact per-episode step proportions are preserved.
    """
    remaining = _rounded_counts(steps, POLICY_WEIGHTS)
    result = []
    start = 0
    while remaining.sum():
        policy = int(rng.choice(4, p=remaining / remaining.sum()))
        length = min(int(rng.integers(2, 17)), int(remaining[policy]))
        if remaining[policy] - length == 1:
            length = length - 1 if length > 2 else length + 1
        result.append((policy, start, length))
        remaining[policy] -= length
        start += length
    return result


def _feedback_action(sim):
    if sim.family == "thermal":
        return np.array([np.clip((22 - sim.state[0]) * 2.5, -15, 15)])
    if sim.family == "tank":
        fraction = sim.state[0] / sim.parameters["capacity"]
        return np.array([0.9, np.clip(0.4 + (0.55 - fraction) * 2, 0, 1)])
    return np.array([np.clip((0.55 - sim.state[0]) * 15, -20, 20)])


def _segment_actions(sim, policy, length, rng):
    names = role_fields(sim.schema, "action")
    bounds = np.array(
        [(FIELD_SPECS[FIELD_INDEX[n]].minimum, FIELD_SPECS[FIELD_INDEX[n]].maximum) for n in names]
    )
    low, high = bounds[:, 0], bounds[:, 1]
    if policy == 0:
        return np.tile(rng.uniform(low, high), (length, 1))
    if policy == 1:
        start, end = rng.uniform(low, high), rng.uniform(low, high)
        if rng.random() < 0.5 and length > 1:
            return np.linspace(start, end, length)  # constant-rate ramp
        result = np.tile(start, (length, 1))
        if length > 1:
            result[int(rng.integers(1, length)) :] = end  # sustained step intervention
        return result
    if policy == 3:
        return np.tile(np.where(rng.integers(0, 2, len(names)), high, low), (length, 1))
    return None  # feedback must use each newly observed simulator state


def generate_episode(family, system_id, days=7, seed=0, split=None, missing_probability=0.02):
    from tfm.simulators import make_simulator

    steps = round(days * 24 * 60 / DT_MINUTES)
    if steps < 1 or not 0 <= missing_probability < 1:
        raise ValueError("days must be positive; missing_probability in [0,1)")
    streams = np.random.SeedSequence(seed).spawn(4)
    sim = make_simulator(family, int(streams[0].generate_state(1)[0]))
    exogenous = generate_exogenous(family, steps, int(streams[1].generate_state(1)[0]))
    rng, mask_rng = np.random.default_rng(streams[2]), np.random.default_rng(streams[3])
    values = np.zeros((steps + 1, NUM_FIELDS))
    actions = np.zeros((steps, NUM_FIELDS))
    mask = np.zeros_like(values, dtype=bool)
    states = [FIELD_INDEX[n] for n in role_fields(sim.schema, "state")]
    controls = [FIELD_INDEX[n] for n in role_fields(sim.schema, "action")]
    exposures = [FIELD_INDEX[n] for n in role_fields(sim.schema, "exogenous")]
    policies = np.zeros(steps, dtype=np.int8)
    segment_ids = np.zeros(steps, dtype=np.int32)
    values[0, states] = sim.state
    for segment_id, (policy, start, length) in enumerate(policy_segments(steps, rng)):
        planned = _segment_actions(sim, policy, length, rng)
        policies[start : start + length] = policy
        segment_ids[start : start + length] = segment_id
        for offset in range(length):
            t = start + offset
            compact = _feedback_action(sim) if policy == 2 else planned[offset]
            actions[t, controls] = compact
            values[t, controls] = compact
            values[t, exposures] = exogenous[t, exposures]
            values[t + 1, states] = sim.step(actions[t], exogenous[t])
    mask[:, states] = mask_rng.random((steps + 1, len(states))) >= missing_probability
    mask[:-1, controls + exposures] = True
    return Episode(
        system_id,
        family,
        sim.schema,
        np.arange(steps + 1) * DT_MINUTES,
        values,
        mask,
        actions,
        exogenous,
        sim.parameters,
        sim.seed,
        split,
        policies,
        segment_ids,
    )


def generate_dataset(
    output, systems=1200, days=7, seed=0, families=("thermal", "tank", "battery"), parquet=False
):
    output = Path(output)
    output.mkdir(parents=True, exist_ok=True)
    families = tuple(families)
    if not families or any(f not in SCHEMAS for f in families) or systems < 3 * len(families):
        raise ValueError("Need known families and at least 3 systems per family for disjoint partitions")
    manifest = {
        "generator_version": GENERATOR_VERSION,
        "seed": seed,
        "dt_minutes": DT_MINUTES,
        "systems": systems,
        "days": days,
        "policy_names": list(POLICY_NAMES),
        "policy_weights": list(POLICY_WEIGHTS),
        "split_unit": "system_id",
        "split_weights": [0.70, 0.15, 0.15],
        "policy_segment_length_steps": [2, 16],
        "policy_segment_quota_tail_may_be_one_step": True,
        "episodes": [],
    }
    assigned = [families[i % len(families)] for i in range(systems)]
    splits = split_systems(assigned, seed)
    for i, family in enumerate(assigned):
        episode_seed = int(np.random.SeedSequence([seed, i]).generate_state(1)[0])
        system_id = f"{family}-{i:05d}"
        episode = generate_episode(family, system_id, days, episode_seed, splits[i])
        path = f"episodes/{system_id}.npz"
        save_episode(episode, output / path)
        entry = {
            "path": path,
            "system_id": system_id,
            "family_id": family,
            "split": splits[i],
            "seed": episode_seed,
        }
        if parquet:
            parquet_path = output / "parquet" / f"{system_id}.parquet"
            parquet_path.parent.mkdir(parents=True, exist_ok=True)
            save_episode_parquet(episode, parquet_path)
            entry["parquet_path"] = str(parquet_path.relative_to(output))
        manifest["episodes"].append(entry)
    manifest["split_system_ids"] = {
        split: sorted(e["system_id"] for e in manifest["episodes"] if e["split"] == split)
        for split in ("train", "validation", "test")
    }
    manifest["normalization_path"] = "normalization.json"
    (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
    from .normalization import Normalization

    Normalization.fit(load_manifest_episodes(output, "train")).save(output / "normalization.json")
    return manifest


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", required=True)
    parser.add_argument("--systems", type=int, default=1200)
    parser.add_argument("--days", type=float, default=7)
    parser.add_argument("--seed", type=int, default=0)
    parser.add_argument("--families", nargs="+", choices=tuple(SCHEMAS), default=list(SCHEMAS))
    parser.add_argument("--parquet", action="store_true")
    args = parser.parse_args()
    manifest = generate_dataset(**vars(args))
    print(f"Generated {len(manifest['episodes'])} full episodes at {args.output}")


if __name__ == "__main__":
    main()