Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/data/external.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 8.6 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/external.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/data/external.py
-
curl -L -o external.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/external.py
8.6 kB
| """Isolated native collectors. Raw external names/units require a validated mapping before training. | |
| These collectors deliberately never claim arbitrary external observations are the | |
| synthetic vocabulary. They preserve native controls and provenance for milestone 5. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from urllib.request import Request, urlopen | |
| import numpy as np | |
| def _save_native(path, arrays, metadata): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| arrays = {k: np.asarray(v) for k, v in arrays.items()} | |
| if any(v.dtype == object for v in arrays.values()): | |
| raise ValueError("Native arrays must be numerical; flatten structured observations explicitly") | |
| np.savez_compressed(path, **arrays) | |
| metadata = { | |
| **metadata, | |
| "collector_version": "tfm-native-v1", | |
| "schema_mapping_validated": False, | |
| "normalization_status": "not fitted; keep external and synthetic sources separate", | |
| } | |
| path.with_suffix(".json").write_text(json.dumps(metadata, indent=2, allow_nan=False)) | |
| return metadata | |
| def collect_citylearn(env, policy, path, *, system_id, steps=672, seed=42): | |
| """Collect an instantiated CityLearn 3 list-interface environment. | |
| policy(observations, action_names, action_spaces) -> one vector per enabled | |
| agent. Never invent thermostat controls or reinterpret storage fractions as kW. | |
| """ | |
| action_names = env.action_names | |
| observation_names = env.observation_names | |
| spaces = env.action_space | |
| if not isinstance(spaces, list): | |
| raise ValueError("Use CityLearn's list interface or provide a dedicated dict-interface adapter") | |
| if any(len(names) != space.shape[0] for names, space in zip(action_names, spaces)): | |
| raise ValueError("Native action names and action-space dimensions differ") | |
| reset = env.reset(seed=seed) | |
| observations = reset[0] if isinstance(reset, tuple) else reset | |
| observed = [[np.asarray(o, dtype=float)] for o in observations] | |
| controls = [[] for _ in spaces] | |
| for _ in range(steps): | |
| actions = policy(observations, action_names, spaces) | |
| if len(actions) != len(spaces): | |
| raise ValueError("Policy must provide every enabled agent's native actions") | |
| validated = [] | |
| for i, (action, space) in enumerate(zip(actions, spaces)): | |
| value = np.asarray(action, dtype=space.dtype) | |
| if not space.contains(value) or not np.isfinite(value).all(): | |
| raise ValueError(f"Invalid native CityLearn action for agent {i}") | |
| validated.append(value) | |
| controls[i].append(value) | |
| observations, _, terminated, truncated, _ = env.step(validated) | |
| for seq, obs in zip(observed, observations): | |
| seq.append(np.asarray(obs, dtype=float)) | |
| if terminated or truncated: | |
| break | |
| arrays = { | |
| **{f"observations_{i}": x for i, x in enumerate(observed)}, | |
| **{f"actions_{i}": x for i, x in enumerate(controls)}, | |
| } | |
| return _save_native( | |
| path, | |
| arrays, | |
| { | |
| "source": "CityLearn", | |
| "system_id": str(system_id), | |
| "seed": seed, | |
| "action_names": action_names, | |
| "observation_names": observation_names, | |
| "seconds_per_time_step": float(env.seconds_per_time_step), | |
| "action_bounds": [{"low": s.low.tolist(), "high": s.high.tolist()} for s in spaces], | |
| "exogenous_status": "derive native weather/forecast separately; no future states as exogenous", | |
| }, | |
| ) | |
| class BoptestClient: | |
| """Direct instance REST client. Start/select an instance outside this collector. | |
| endpoint must already identify the instance (hosted service routes may require | |
| their own testid prefix). Native input/measurement units are preserved. | |
| """ | |
| def __init__(self, endpoint, timeout=30): | |
| self.endpoint = endpoint.rstrip("/") | |
| self.timeout = timeout | |
| def call(self, method, path, payload=None): | |
| request = Request( | |
| self.endpoint + "/" + path.lstrip("/"), | |
| data=json.dumps(payload).encode() if payload is not None else None, | |
| headers={"Content-Type": "application/json"}, | |
| method=method, | |
| ) | |
| with urlopen(request, timeout=self.timeout) as response: | |
| result = json.load(response) | |
| if isinstance(result, dict) and "payload" in result: | |
| if result.get("status", 200) not in (200, 201): | |
| raise ValueError(f"BOPTEST rejected {path}: {result.get('message')}") | |
| return result["payload"] | |
| return result | |
| def collect_boptest(client, policy, path, *, system_id, start_time=0, warmup_period=86400, steps=672): | |
| """Preserve native inputs and specified scenario/weather forecast snapshots. | |
| policy(measurement,input_metadata) returns native overrides (including activate | |
| flags where required). Forecast snapshots are recorded before each intervention. | |
| """ | |
| inputs = client.call("GET", "inputs") | |
| measurements = client.call("GET", "measurements") | |
| forecast_points = client.call("GET", "forecast_points") | |
| client.call("PUT", "step", {"step": 900}) | |
| measurement = client.call("PUT", "initialize", {"start_time": start_time, "warmup_period": warmup_period}) | |
| states, actions, forecasts = [measurement], [], [] | |
| for _ in range(steps): | |
| action = policy(measurement, inputs) | |
| for name, value in action.items(): | |
| if name not in inputs: | |
| raise ValueError(f"Unknown BOPTEST input {name}") | |
| spec = inputs[name] | |
| if ( | |
| not np.isfinite(value) | |
| or value < spec.get("Minimum", -np.inf) | |
| or value > spec.get("Maximum", np.inf) | |
| ): | |
| raise ValueError(f"Invalid BOPTEST input {name}") | |
| forecasts.append( | |
| client.call( | |
| "PUT", "forecast", {"point_names": list(forecast_points), "horizon": 7200, "interval": 900} | |
| ) | |
| ) | |
| measurement = client.call("POST", "advance", action) | |
| states.append(measurement) | |
| actions.append(action) | |
| # Native dictionary snapshots retain scenario-specific units and optional inputs. | |
| target = Path(path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| target.write_text( | |
| json.dumps( | |
| { | |
| "source": "BOPTEST", | |
| "collector_version": "tfm-native-v1", | |
| "system_id": str(system_id), | |
| "dt_seconds": 900, | |
| "inputs": inputs, | |
| "measurements": measurements, | |
| "forecast_points": forecast_points, | |
| "states": states, | |
| "actions": actions, | |
| "forecasts": forecasts, | |
| "schema_mapping_validated": False, | |
| }, | |
| indent=2, | |
| allow_nan=False, | |
| ) | |
| ) | |
| return target | |
| def collect_minari(dataset, path, *, system_id_for_episode): | |
| """Collect numerical offline episodes; caller supplies physical-instance IDs. | |
| An episode ID is not evidence of a distinct physical system. This is why an | |
| explicit instance grouping function is required before any dataset split. | |
| """ | |
| output = Path(path) | |
| output.mkdir(parents=True, exist_ok=True) | |
| collected = [] | |
| for episode in dataset.iterate_episodes(): | |
| if isinstance(episode.observations, dict) or isinstance(episode.actions, dict): | |
| raise ValueError("Structured Minari observations/actions require explicit numerical flattening") | |
| sid = system_id_for_episode(episode) | |
| if sid is None: | |
| raise ValueError( | |
| "Physical system grouping must be supplied; episode IDs cannot imply unseen systems" | |
| ) | |
| metadata = _save_native( | |
| output / f"episode-{episode.id}.npz", | |
| { | |
| "observations": episode.observations, | |
| "actions": episode.actions, | |
| "terminations": episode.terminations, | |
| "truncations": episode.truncations, | |
| }, | |
| { | |
| "source": "Minari", | |
| "dataset_id": dataset.spec.dataset_id, | |
| "episode_id": int(episode.id), | |
| "system_id": str(sid), | |
| "intervention_coverage": "offline observed actions only", | |
| }, | |
| ) | |
| collected.append(metadata) | |
| (output / "native_manifest.json").write_text(json.dumps(collected, indent=2)) | |
| return collected | |