"""Seeded mixed-policy episode generation and stratified system partitions. Exogenous values are forecast inputs at transition t, never future state sensors. Thermal/battery temperatures follow daily sinusoids plus correlated innovations; solar peaks in daytime, occupancy and tank demand follow daily schedules. """ import argparse import json from pathlib import Path import numpy as np from .episodes import Episode, load_manifest_episodes, save_episode, save_episode_parquet from .schema import DT_MINUTES, FIELD_INDEX, FIELD_SPECS, GENERATOR_VERSION, NUM_FIELDS, SCHEMAS, role_fields POLICY_NAMES = ("random_piecewise", "structured_step_ramp", "feedback", "boundary") POLICY_WEIGHTS = (0.35, 0.30, 0.25, 0.10) def generate_exogenous(family, steps, seed): rng = np.random.default_rng(seed) hour = np.arange(steps) * DT_MINUTES / 60 % 24 result = np.zeros((steps, NUM_FIELDS)) innovation = np.zeros(steps) for t in range(1, steps): innovation[t] = 0.94 * innovation[t - 1] + rng.normal(0, 0.25) daily = rng.uniform(8, 24) + rng.uniform(3, 9) * np.sin((hour - 9) * np.pi / 12) + innovation if family == "thermal": result[:, FIELD_INDEX["outdoor_temp"]] = daily result[:, FIELD_INDEX["solar_gain"]] = rng.uniform(0.5, 3) * np.maximum( 0, np.sin((hour - 6) * np.pi / 12) ) result[:, FIELD_INDEX["occupancy_gain"]] = 0.12 + ((hour >= 17) | (hour < 8)) * rng.uniform(0.1, 0.7) elif family == "tank": result[:, FIELD_INDEX["demand"]] = np.clip( 0.4 + 2 * np.exp(-(((hour - 8) / 2) ** 2)) + 2 * np.exp(-(((hour - 19) / 3) ** 2)) + rng.normal(0, 0.15, steps), 0, 10, ) result[:, FIELD_INDEX["inlet_temp"]] = 13 + 0.15 * daily elif family == "battery": result[:, FIELD_INDEX["ambient_temp"]] = daily else: raise ValueError(f"Unknown family {family}") return result def _rounded_counts(size, weights): """Largest-remainder integer quotas, preserving the requested total.""" expected = np.asarray(weights, dtype=float) * size counts = np.floor(expected).astype(int) order = np.argsort(-(expected - counts), kind="stable") counts[order[: size - int(counts.sum())]] += 1 return counts def split_systems(assigned_families, seed=0): """Deterministic stratified 70/15/15 system split with global integer quotas. Each family has at least one system in each partition. At small sizes this constraint takes priority over fractional quotas. Equal rounding ties go to validation before test, so 90 systems split 63/14/13, and 1200 split 840/180/180. """ assigned_families = list(assigned_families) families = tuple(dict.fromkeys(assigned_families)) if not families: raise ValueError("Cannot split an empty system list") groups = [np.array([i for i, f in enumerate(assigned_families) if f == family]) for family in families] if any(len(ids) < 3 for ids in groups): raise ValueError("Stratification requires at least three systems per family") weights = (0.70, 0.15, 0.15) target = _rounded_counts(len(assigned_families), weights) counts = np.array([np.maximum(1, np.floor(len(ids) * np.array(weights)).astype(int)) for ids in groups]) for row, ids in zip(counts, groups): while row.sum() > len(ids): eligible = np.where(row > 1, row, -1) row[int(np.argmax(eligible))] -= 1 for row, ids in zip(counts, groups): while row.sum() < len(ids): deficits = target - counts.sum(axis=0) row[int(np.argmax(deficits))] += 1 rng = np.random.default_rng(seed) splits = {} for ids, (n_train, n_val, _) in zip(groups, counts): rng.shuffle(ids) for split, selected in ( ("train", ids[:n_train]), ("validation", ids[n_train : n_train + n_val]), ("test", ids[n_train + n_val :]), ): splits.update({int(i): split for i in selected}) return splits def policy_segments(steps, rng): """Random 2..16-step segments selected by remaining 35/30/25/10 quotas. Segment boundaries remain explicit even for adjacent segments of the same policy. Sampled lengths are adjusted at quota tails to avoid one-step holds. Only very short episodes whose entire policy quota is one step require a one-step truncated segment. Exact per-episode step proportions are preserved. """ remaining = _rounded_counts(steps, POLICY_WEIGHTS) result = [] start = 0 while remaining.sum(): policy = int(rng.choice(4, p=remaining / remaining.sum())) length = min(int(rng.integers(2, 17)), int(remaining[policy])) if remaining[policy] - length == 1: length = length - 1 if length > 2 else length + 1 result.append((policy, start, length)) remaining[policy] -= length start += length return result def _feedback_action(sim): if sim.family == "thermal": return np.array([np.clip((22 - sim.state[0]) * 2.5, -15, 15)]) if sim.family == "tank": fraction = sim.state[0] / sim.parameters["capacity"] return np.array([0.9, np.clip(0.4 + (0.55 - fraction) * 2, 0, 1)]) return np.array([np.clip((0.55 - sim.state[0]) * 15, -20, 20)]) def _segment_actions(sim, policy, length, rng): names = role_fields(sim.schema, "action") bounds = np.array( [(FIELD_SPECS[FIELD_INDEX[n]].minimum, FIELD_SPECS[FIELD_INDEX[n]].maximum) for n in names] ) low, high = bounds[:, 0], bounds[:, 1] if policy == 0: return np.tile(rng.uniform(low, high), (length, 1)) if policy == 1: start, end = rng.uniform(low, high), rng.uniform(low, high) if rng.random() < 0.5 and length > 1: return np.linspace(start, end, length) # constant-rate ramp result = np.tile(start, (length, 1)) if length > 1: result[int(rng.integers(1, length)) :] = end # sustained step intervention return result if policy == 3: return np.tile(np.where(rng.integers(0, 2, len(names)), high, low), (length, 1)) return None # feedback must use each newly observed simulator state def generate_episode(family, system_id, days=7, seed=0, split=None, missing_probability=0.02): from tfm.simulators import make_simulator steps = round(days * 24 * 60 / DT_MINUTES) if steps < 1 or not 0 <= missing_probability < 1: raise ValueError("days must be positive; missing_probability in [0,1)") streams = np.random.SeedSequence(seed).spawn(4) sim = make_simulator(family, int(streams[0].generate_state(1)[0])) exogenous = generate_exogenous(family, steps, int(streams[1].generate_state(1)[0])) rng, mask_rng = np.random.default_rng(streams[2]), np.random.default_rng(streams[3]) values = np.zeros((steps + 1, NUM_FIELDS)) actions = np.zeros((steps, NUM_FIELDS)) mask = np.zeros_like(values, dtype=bool) states = [FIELD_INDEX[n] for n in role_fields(sim.schema, "state")] controls = [FIELD_INDEX[n] for n in role_fields(sim.schema, "action")] exposures = [FIELD_INDEX[n] for n in role_fields(sim.schema, "exogenous")] policies = np.zeros(steps, dtype=np.int8) segment_ids = np.zeros(steps, dtype=np.int32) values[0, states] = sim.state for segment_id, (policy, start, length) in enumerate(policy_segments(steps, rng)): planned = _segment_actions(sim, policy, length, rng) policies[start : start + length] = policy segment_ids[start : start + length] = segment_id for offset in range(length): t = start + offset compact = _feedback_action(sim) if policy == 2 else planned[offset] actions[t, controls] = compact values[t, controls] = compact values[t, exposures] = exogenous[t, exposures] values[t + 1, states] = sim.step(actions[t], exogenous[t]) mask[:, states] = mask_rng.random((steps + 1, len(states))) >= missing_probability mask[:-1, controls + exposures] = True return Episode( system_id, family, sim.schema, np.arange(steps + 1) * DT_MINUTES, values, mask, actions, exogenous, sim.parameters, sim.seed, split, policies, segment_ids, ) def generate_dataset( output, systems=1200, days=7, seed=0, families=("thermal", "tank", "battery"), parquet=False ): output = Path(output) output.mkdir(parents=True, exist_ok=True) families = tuple(families) if not families or any(f not in SCHEMAS for f in families) or systems < 3 * len(families): raise ValueError("Need known families and at least 3 systems per family for disjoint partitions") manifest = { "generator_version": GENERATOR_VERSION, "seed": seed, "dt_minutes": DT_MINUTES, "systems": systems, "days": days, "policy_names": list(POLICY_NAMES), "policy_weights": list(POLICY_WEIGHTS), "split_unit": "system_id", "split_weights": [0.70, 0.15, 0.15], "policy_segment_length_steps": [2, 16], "policy_segment_quota_tail_may_be_one_step": True, "episodes": [], } assigned = [families[i % len(families)] for i in range(systems)] splits = split_systems(assigned, seed) for i, family in enumerate(assigned): episode_seed = int(np.random.SeedSequence([seed, i]).generate_state(1)[0]) system_id = f"{family}-{i:05d}" episode = generate_episode(family, system_id, days, episode_seed, splits[i]) path = f"episodes/{system_id}.npz" save_episode(episode, output / path) entry = { "path": path, "system_id": system_id, "family_id": family, "split": splits[i], "seed": episode_seed, } if parquet: parquet_path = output / "parquet" / f"{system_id}.parquet" parquet_path.parent.mkdir(parents=True, exist_ok=True) save_episode_parquet(episode, parquet_path) entry["parquet_path"] = str(parquet_path.relative_to(output)) manifest["episodes"].append(entry) manifest["split_system_ids"] = { split: sorted(e["system_id"] for e in manifest["episodes"] if e["split"] == split) for split in ("train", "validation", "test") } manifest["normalization_path"] = "normalization.json" (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") from .normalization import Normalization Normalization.fit(load_manifest_episodes(output, "train")).save(output / "normalization.json") return manifest def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True) parser.add_argument("--systems", type=int, default=1200) parser.add_argument("--days", type=float, default=7) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--families", nargs="+", choices=tuple(SCHEMAS), default=list(SCHEMAS)) parser.add_argument("--parquet", action="store_true") args = parser.parse_args() manifest = generate_dataset(**vars(args)) print(f"Generated {len(manifest['episodes'])} full episodes at {args.output}") if __name__ == "__main__": main()