maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
11.5 kB
"""Seeded mixed-policy episode generation and stratified system partitions.
Exogenous values are forecast inputs at transition t, never future state sensors.
Thermal/battery temperatures follow daily sinusoids plus correlated innovations;
solar peaks in daytime, occupancy and tank demand follow daily schedules.
"""
import argparse
import json
from pathlib import Path
import numpy as np
from .episodes import Episode, load_manifest_episodes, save_episode, save_episode_parquet
from .schema import DT_MINUTES, FIELD_INDEX, FIELD_SPECS, GENERATOR_VERSION, NUM_FIELDS, SCHEMAS, role_fields
POLICY_NAMES = ("random_piecewise", "structured_step_ramp", "feedback", "boundary")
POLICY_WEIGHTS = (0.35, 0.30, 0.25, 0.10)
def generate_exogenous(family, steps, seed):
rng = np.random.default_rng(seed)
hour = np.arange(steps) * DT_MINUTES / 60 % 24
result = np.zeros((steps, NUM_FIELDS))
innovation = np.zeros(steps)
for t in range(1, steps):
innovation[t] = 0.94 * innovation[t - 1] + rng.normal(0, 0.25)
daily = rng.uniform(8, 24) + rng.uniform(3, 9) * np.sin((hour - 9) * np.pi / 12) + innovation
if family == "thermal":
result[:, FIELD_INDEX["outdoor_temp"]] = daily
result[:, FIELD_INDEX["solar_gain"]] = rng.uniform(0.5, 3) * np.maximum(
0, np.sin((hour - 6) * np.pi / 12)
)
result[:, FIELD_INDEX["occupancy_gain"]] = 0.12 + ((hour >= 17) | (hour < 8)) * rng.uniform(0.1, 0.7)
elif family == "tank":
result[:, FIELD_INDEX["demand"]] = np.clip(
0.4
+ 2 * np.exp(-(((hour - 8) / 2) ** 2))
+ 2 * np.exp(-(((hour - 19) / 3) ** 2))
+ rng.normal(0, 0.15, steps),
0,
10,
)
result[:, FIELD_INDEX["inlet_temp"]] = 13 + 0.15 * daily
elif family == "battery":
result[:, FIELD_INDEX["ambient_temp"]] = daily
else:
raise ValueError(f"Unknown family {family}")
return result
def _rounded_counts(size, weights):
"""Largest-remainder integer quotas, preserving the requested total."""
expected = np.asarray(weights, dtype=float) * size
counts = np.floor(expected).astype(int)
order = np.argsort(-(expected - counts), kind="stable")
counts[order[: size - int(counts.sum())]] += 1
return counts
def split_systems(assigned_families, seed=0):
"""Deterministic stratified 70/15/15 system split with global integer quotas.
Each family has at least one system in each partition. At small sizes this
constraint takes priority over fractional quotas. Equal rounding ties go to
validation before test, so 90 systems split 63/14/13, and 1200 split 840/180/180.
"""
assigned_families = list(assigned_families)
families = tuple(dict.fromkeys(assigned_families))
if not families:
raise ValueError("Cannot split an empty system list")
groups = [np.array([i for i, f in enumerate(assigned_families) if f == family]) for family in families]
if any(len(ids) < 3 for ids in groups):
raise ValueError("Stratification requires at least three systems per family")
weights = (0.70, 0.15, 0.15)
target = _rounded_counts(len(assigned_families), weights)
counts = np.array([np.maximum(1, np.floor(len(ids) * np.array(weights)).astype(int)) for ids in groups])
for row, ids in zip(counts, groups):
while row.sum() > len(ids):
eligible = np.where(row > 1, row, -1)
row[int(np.argmax(eligible))] -= 1
for row, ids in zip(counts, groups):
while row.sum() < len(ids):
deficits = target - counts.sum(axis=0)
row[int(np.argmax(deficits))] += 1
rng = np.random.default_rng(seed)
splits = {}
for ids, (n_train, n_val, _) in zip(groups, counts):
rng.shuffle(ids)
for split, selected in (
("train", ids[:n_train]),
("validation", ids[n_train : n_train + n_val]),
("test", ids[n_train + n_val :]),
):
splits.update({int(i): split for i in selected})
return splits
def policy_segments(steps, rng):
"""Random 2..16-step segments selected by remaining 35/30/25/10 quotas.
Segment boundaries remain explicit even for adjacent segments of the same
policy. Sampled lengths are adjusted at quota tails to avoid one-step holds.
Only very short episodes whose entire policy quota is one step require a
one-step truncated segment. Exact per-episode step proportions are preserved.
"""
remaining = _rounded_counts(steps, POLICY_WEIGHTS)
result = []
start = 0
while remaining.sum():
policy = int(rng.choice(4, p=remaining / remaining.sum()))
length = min(int(rng.integers(2, 17)), int(remaining[policy]))
if remaining[policy] - length == 1:
length = length - 1 if length > 2 else length + 1
result.append((policy, start, length))
remaining[policy] -= length
start += length
return result
def _feedback_action(sim):
if sim.family == "thermal":
return np.array([np.clip((22 - sim.state[0]) * 2.5, -15, 15)])
if sim.family == "tank":
fraction = sim.state[0] / sim.parameters["capacity"]
return np.array([0.9, np.clip(0.4 + (0.55 - fraction) * 2, 0, 1)])
return np.array([np.clip((0.55 - sim.state[0]) * 15, -20, 20)])
def _segment_actions(sim, policy, length, rng):
names = role_fields(sim.schema, "action")
bounds = np.array(
[(FIELD_SPECS[FIELD_INDEX[n]].minimum, FIELD_SPECS[FIELD_INDEX[n]].maximum) for n in names]
)
low, high = bounds[:, 0], bounds[:, 1]
if policy == 0:
return np.tile(rng.uniform(low, high), (length, 1))
if policy == 1:
start, end = rng.uniform(low, high), rng.uniform(low, high)
if rng.random() < 0.5 and length > 1:
return np.linspace(start, end, length) # constant-rate ramp
result = np.tile(start, (length, 1))
if length > 1:
result[int(rng.integers(1, length)) :] = end # sustained step intervention
return result
if policy == 3:
return np.tile(np.where(rng.integers(0, 2, len(names)), high, low), (length, 1))
return None # feedback must use each newly observed simulator state
def generate_episode(family, system_id, days=7, seed=0, split=None, missing_probability=0.02):
from tfm.simulators import make_simulator
steps = round(days * 24 * 60 / DT_MINUTES)
if steps < 1 or not 0 <= missing_probability < 1:
raise ValueError("days must be positive; missing_probability in [0,1)")
streams = np.random.SeedSequence(seed).spawn(4)
sim = make_simulator(family, int(streams[0].generate_state(1)[0]))
exogenous = generate_exogenous(family, steps, int(streams[1].generate_state(1)[0]))
rng, mask_rng = np.random.default_rng(streams[2]), np.random.default_rng(streams[3])
values = np.zeros((steps + 1, NUM_FIELDS))
actions = np.zeros((steps, NUM_FIELDS))
mask = np.zeros_like(values, dtype=bool)
states = [FIELD_INDEX[n] for n in role_fields(sim.schema, "state")]
controls = [FIELD_INDEX[n] for n in role_fields(sim.schema, "action")]
exposures = [FIELD_INDEX[n] for n in role_fields(sim.schema, "exogenous")]
policies = np.zeros(steps, dtype=np.int8)
segment_ids = np.zeros(steps, dtype=np.int32)
values[0, states] = sim.state
for segment_id, (policy, start, length) in enumerate(policy_segments(steps, rng)):
planned = _segment_actions(sim, policy, length, rng)
policies[start : start + length] = policy
segment_ids[start : start + length] = segment_id
for offset in range(length):
t = start + offset
compact = _feedback_action(sim) if policy == 2 else planned[offset]
actions[t, controls] = compact
values[t, controls] = compact
values[t, exposures] = exogenous[t, exposures]
values[t + 1, states] = sim.step(actions[t], exogenous[t])
mask[:, states] = mask_rng.random((steps + 1, len(states))) >= missing_probability
mask[:-1, controls + exposures] = True
return Episode(
system_id,
family,
sim.schema,
np.arange(steps + 1) * DT_MINUTES,
values,
mask,
actions,
exogenous,
sim.parameters,
sim.seed,
split,
policies,
segment_ids,
)
def generate_dataset(
output, systems=1200, days=7, seed=0, families=("thermal", "tank", "battery"), parquet=False
):
output = Path(output)
output.mkdir(parents=True, exist_ok=True)
families = tuple(families)
if not families or any(f not in SCHEMAS for f in families) or systems < 3 * len(families):
raise ValueError("Need known families and at least 3 systems per family for disjoint partitions")
manifest = {
"generator_version": GENERATOR_VERSION,
"seed": seed,
"dt_minutes": DT_MINUTES,
"systems": systems,
"days": days,
"policy_names": list(POLICY_NAMES),
"policy_weights": list(POLICY_WEIGHTS),
"split_unit": "system_id",
"split_weights": [0.70, 0.15, 0.15],
"policy_segment_length_steps": [2, 16],
"policy_segment_quota_tail_may_be_one_step": True,
"episodes": [],
}
assigned = [families[i % len(families)] for i in range(systems)]
splits = split_systems(assigned, seed)
for i, family in enumerate(assigned):
episode_seed = int(np.random.SeedSequence([seed, i]).generate_state(1)[0])
system_id = f"{family}-{i:05d}"
episode = generate_episode(family, system_id, days, episode_seed, splits[i])
path = f"episodes/{system_id}.npz"
save_episode(episode, output / path)
entry = {
"path": path,
"system_id": system_id,
"family_id": family,
"split": splits[i],
"seed": episode_seed,
}
if parquet:
parquet_path = output / "parquet" / f"{system_id}.parquet"
parquet_path.parent.mkdir(parents=True, exist_ok=True)
save_episode_parquet(episode, parquet_path)
entry["parquet_path"] = str(parquet_path.relative_to(output))
manifest["episodes"].append(entry)
manifest["split_system_ids"] = {
split: sorted(e["system_id"] for e in manifest["episodes"] if e["split"] == split)
for split in ("train", "validation", "test")
}
manifest["normalization_path"] = "normalization.json"
(output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
from .normalization import Normalization
Normalization.fit(load_manifest_episodes(output, "train")).save(output / "normalization.json")
return manifest
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", required=True)
parser.add_argument("--systems", type=int, default=1200)
parser.add_argument("--days", type=float, default=7)
parser.add_argument("--seed", type=int, default=0)
parser.add_argument("--families", nargs="+", choices=tuple(SCHEMAS), default=list(SCHEMAS))
parser.add_argument("--parquet", action="store_true")
args = parser.parse_args()
manifest = generate_dataset(**vars(args))
print(f"Generated {len(manifest['episodes'])} full episodes at {args.output}")
if __name__ == "__main__":
main()