Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/data/generate.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/generate.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/data/generate.py
-
curl -L -o generate.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/generate.py
11.5 kB
| """Seeded mixed-policy episode generation and stratified system partitions. | |
| Exogenous values are forecast inputs at transition t, never future state sensors. | |
| Thermal/battery temperatures follow daily sinusoids plus correlated innovations; | |
| solar peaks in daytime, occupancy and tank demand follow daily schedules. | |
| """ | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from .episodes import Episode, load_manifest_episodes, save_episode, save_episode_parquet | |
| from .schema import DT_MINUTES, FIELD_INDEX, FIELD_SPECS, GENERATOR_VERSION, NUM_FIELDS, SCHEMAS, role_fields | |
| POLICY_NAMES = ("random_piecewise", "structured_step_ramp", "feedback", "boundary") | |
| POLICY_WEIGHTS = (0.35, 0.30, 0.25, 0.10) | |
| def generate_exogenous(family, steps, seed): | |
| rng = np.random.default_rng(seed) | |
| hour = np.arange(steps) * DT_MINUTES / 60 % 24 | |
| result = np.zeros((steps, NUM_FIELDS)) | |
| innovation = np.zeros(steps) | |
| for t in range(1, steps): | |
| innovation[t] = 0.94 * innovation[t - 1] + rng.normal(0, 0.25) | |
| daily = rng.uniform(8, 24) + rng.uniform(3, 9) * np.sin((hour - 9) * np.pi / 12) + innovation | |
| if family == "thermal": | |
| result[:, FIELD_INDEX["outdoor_temp"]] = daily | |
| result[:, FIELD_INDEX["solar_gain"]] = rng.uniform(0.5, 3) * np.maximum( | |
| 0, np.sin((hour - 6) * np.pi / 12) | |
| ) | |
| result[:, FIELD_INDEX["occupancy_gain"]] = 0.12 + ((hour >= 17) | (hour < 8)) * rng.uniform(0.1, 0.7) | |
| elif family == "tank": | |
| result[:, FIELD_INDEX["demand"]] = np.clip( | |
| 0.4 | |
| + 2 * np.exp(-(((hour - 8) / 2) ** 2)) | |
| + 2 * np.exp(-(((hour - 19) / 3) ** 2)) | |
| + rng.normal(0, 0.15, steps), | |
| 0, | |
| 10, | |
| ) | |
| result[:, FIELD_INDEX["inlet_temp"]] = 13 + 0.15 * daily | |
| elif family == "battery": | |
| result[:, FIELD_INDEX["ambient_temp"]] = daily | |
| else: | |
| raise ValueError(f"Unknown family {family}") | |
| return result | |
| def _rounded_counts(size, weights): | |
| """Largest-remainder integer quotas, preserving the requested total.""" | |
| expected = np.asarray(weights, dtype=float) * size | |
| counts = np.floor(expected).astype(int) | |
| order = np.argsort(-(expected - counts), kind="stable") | |
| counts[order[: size - int(counts.sum())]] += 1 | |
| return counts | |
| def split_systems(assigned_families, seed=0): | |
| """Deterministic stratified 70/15/15 system split with global integer quotas. | |
| Each family has at least one system in each partition. At small sizes this | |
| constraint takes priority over fractional quotas. Equal rounding ties go to | |
| validation before test, so 90 systems split 63/14/13, and 1200 split 840/180/180. | |
| """ | |
| assigned_families = list(assigned_families) | |
| families = tuple(dict.fromkeys(assigned_families)) | |
| if not families: | |
| raise ValueError("Cannot split an empty system list") | |
| groups = [np.array([i for i, f in enumerate(assigned_families) if f == family]) for family in families] | |
| if any(len(ids) < 3 for ids in groups): | |
| raise ValueError("Stratification requires at least three systems per family") | |
| weights = (0.70, 0.15, 0.15) | |
| target = _rounded_counts(len(assigned_families), weights) | |
| counts = np.array([np.maximum(1, np.floor(len(ids) * np.array(weights)).astype(int)) for ids in groups]) | |
| for row, ids in zip(counts, groups): | |
| while row.sum() > len(ids): | |
| eligible = np.where(row > 1, row, -1) | |
| row[int(np.argmax(eligible))] -= 1 | |
| for row, ids in zip(counts, groups): | |
| while row.sum() < len(ids): | |
| deficits = target - counts.sum(axis=0) | |
| row[int(np.argmax(deficits))] += 1 | |
| rng = np.random.default_rng(seed) | |
| splits = {} | |
| for ids, (n_train, n_val, _) in zip(groups, counts): | |
| rng.shuffle(ids) | |
| for split, selected in ( | |
| ("train", ids[:n_train]), | |
| ("validation", ids[n_train : n_train + n_val]), | |
| ("test", ids[n_train + n_val :]), | |
| ): | |
| splits.update({int(i): split for i in selected}) | |
| return splits | |
| def policy_segments(steps, rng): | |
| """Random 2..16-step segments selected by remaining 35/30/25/10 quotas. | |
| Segment boundaries remain explicit even for adjacent segments of the same | |
| policy. Sampled lengths are adjusted at quota tails to avoid one-step holds. | |
| Only very short episodes whose entire policy quota is one step require a | |
| one-step truncated segment. Exact per-episode step proportions are preserved. | |
| """ | |
| remaining = _rounded_counts(steps, POLICY_WEIGHTS) | |
| result = [] | |
| start = 0 | |
| while remaining.sum(): | |
| policy = int(rng.choice(4, p=remaining / remaining.sum())) | |
| length = min(int(rng.integers(2, 17)), int(remaining[policy])) | |
| if remaining[policy] - length == 1: | |
| length = length - 1 if length > 2 else length + 1 | |
| result.append((policy, start, length)) | |
| remaining[policy] -= length | |
| start += length | |
| return result | |
| def _feedback_action(sim): | |
| if sim.family == "thermal": | |
| return np.array([np.clip((22 - sim.state[0]) * 2.5, -15, 15)]) | |
| if sim.family == "tank": | |
| fraction = sim.state[0] / sim.parameters["capacity"] | |
| return np.array([0.9, np.clip(0.4 + (0.55 - fraction) * 2, 0, 1)]) | |
| return np.array([np.clip((0.55 - sim.state[0]) * 15, -20, 20)]) | |
| def _segment_actions(sim, policy, length, rng): | |
| names = role_fields(sim.schema, "action") | |
| bounds = np.array( | |
| [(FIELD_SPECS[FIELD_INDEX[n]].minimum, FIELD_SPECS[FIELD_INDEX[n]].maximum) for n in names] | |
| ) | |
| low, high = bounds[:, 0], bounds[:, 1] | |
| if policy == 0: | |
| return np.tile(rng.uniform(low, high), (length, 1)) | |
| if policy == 1: | |
| start, end = rng.uniform(low, high), rng.uniform(low, high) | |
| if rng.random() < 0.5 and length > 1: | |
| return np.linspace(start, end, length) # constant-rate ramp | |
| result = np.tile(start, (length, 1)) | |
| if length > 1: | |
| result[int(rng.integers(1, length)) :] = end # sustained step intervention | |
| return result | |
| if policy == 3: | |
| return np.tile(np.where(rng.integers(0, 2, len(names)), high, low), (length, 1)) | |
| return None # feedback must use each newly observed simulator state | |
| def generate_episode(family, system_id, days=7, seed=0, split=None, missing_probability=0.02): | |
| from tfm.simulators import make_simulator | |
| steps = round(days * 24 * 60 / DT_MINUTES) | |
| if steps < 1 or not 0 <= missing_probability < 1: | |
| raise ValueError("days must be positive; missing_probability in [0,1)") | |
| streams = np.random.SeedSequence(seed).spawn(4) | |
| sim = make_simulator(family, int(streams[0].generate_state(1)[0])) | |
| exogenous = generate_exogenous(family, steps, int(streams[1].generate_state(1)[0])) | |
| rng, mask_rng = np.random.default_rng(streams[2]), np.random.default_rng(streams[3]) | |
| values = np.zeros((steps + 1, NUM_FIELDS)) | |
| actions = np.zeros((steps, NUM_FIELDS)) | |
| mask = np.zeros_like(values, dtype=bool) | |
| states = [FIELD_INDEX[n] for n in role_fields(sim.schema, "state")] | |
| controls = [FIELD_INDEX[n] for n in role_fields(sim.schema, "action")] | |
| exposures = [FIELD_INDEX[n] for n in role_fields(sim.schema, "exogenous")] | |
| policies = np.zeros(steps, dtype=np.int8) | |
| segment_ids = np.zeros(steps, dtype=np.int32) | |
| values[0, states] = sim.state | |
| for segment_id, (policy, start, length) in enumerate(policy_segments(steps, rng)): | |
| planned = _segment_actions(sim, policy, length, rng) | |
| policies[start : start + length] = policy | |
| segment_ids[start : start + length] = segment_id | |
| for offset in range(length): | |
| t = start + offset | |
| compact = _feedback_action(sim) if policy == 2 else planned[offset] | |
| actions[t, controls] = compact | |
| values[t, controls] = compact | |
| values[t, exposures] = exogenous[t, exposures] | |
| values[t + 1, states] = sim.step(actions[t], exogenous[t]) | |
| mask[:, states] = mask_rng.random((steps + 1, len(states))) >= missing_probability | |
| mask[:-1, controls + exposures] = True | |
| return Episode( | |
| system_id, | |
| family, | |
| sim.schema, | |
| np.arange(steps + 1) * DT_MINUTES, | |
| values, | |
| mask, | |
| actions, | |
| exogenous, | |
| sim.parameters, | |
| sim.seed, | |
| split, | |
| policies, | |
| segment_ids, | |
| ) | |
| def generate_dataset( | |
| output, systems=1200, days=7, seed=0, families=("thermal", "tank", "battery"), parquet=False | |
| ): | |
| output = Path(output) | |
| output.mkdir(parents=True, exist_ok=True) | |
| families = tuple(families) | |
| if not families or any(f not in SCHEMAS for f in families) or systems < 3 * len(families): | |
| raise ValueError("Need known families and at least 3 systems per family for disjoint partitions") | |
| manifest = { | |
| "generator_version": GENERATOR_VERSION, | |
| "seed": seed, | |
| "dt_minutes": DT_MINUTES, | |
| "systems": systems, | |
| "days": days, | |
| "policy_names": list(POLICY_NAMES), | |
| "policy_weights": list(POLICY_WEIGHTS), | |
| "split_unit": "system_id", | |
| "split_weights": [0.70, 0.15, 0.15], | |
| "policy_segment_length_steps": [2, 16], | |
| "policy_segment_quota_tail_may_be_one_step": True, | |
| "episodes": [], | |
| } | |
| assigned = [families[i % len(families)] for i in range(systems)] | |
| splits = split_systems(assigned, seed) | |
| for i, family in enumerate(assigned): | |
| episode_seed = int(np.random.SeedSequence([seed, i]).generate_state(1)[0]) | |
| system_id = f"{family}-{i:05d}" | |
| episode = generate_episode(family, system_id, days, episode_seed, splits[i]) | |
| path = f"episodes/{system_id}.npz" | |
| save_episode(episode, output / path) | |
| entry = { | |
| "path": path, | |
| "system_id": system_id, | |
| "family_id": family, | |
| "split": splits[i], | |
| "seed": episode_seed, | |
| } | |
| if parquet: | |
| parquet_path = output / "parquet" / f"{system_id}.parquet" | |
| parquet_path.parent.mkdir(parents=True, exist_ok=True) | |
| save_episode_parquet(episode, parquet_path) | |
| entry["parquet_path"] = str(parquet_path.relative_to(output)) | |
| manifest["episodes"].append(entry) | |
| manifest["split_system_ids"] = { | |
| split: sorted(e["system_id"] for e in manifest["episodes"] if e["split"] == split) | |
| for split in ("train", "validation", "test") | |
| } | |
| manifest["normalization_path"] = "normalization.json" | |
| (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") | |
| from .normalization import Normalization | |
| Normalization.fit(load_manifest_episodes(output, "train")).save(output / "normalization.json") | |
| return manifest | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--output", required=True) | |
| parser.add_argument("--systems", type=int, default=1200) | |
| parser.add_argument("--days", type=float, default=7) | |
| parser.add_argument("--seed", type=int, default=0) | |
| parser.add_argument("--families", nargs="+", choices=tuple(SCHEMAS), default=list(SCHEMAS)) | |
| parser.add_argument("--parquet", action="store_true") | |
| args = parser.parse_args() | |
| manifest = generate_dataset(**vars(args)) | |
| print(f"Generated {len(manifest['episodes'])} full episodes at {args.output}") | |
| if __name__ == "__main__": | |
| main() | |