maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
8.6 kB
"""Isolated native collectors. Raw external names/units require a validated mapping before training.
These collectors deliberately never claim arbitrary external observations are the
synthetic vocabulary. They preserve native controls and provenance for milestone 5.
"""
from __future__ import annotations
import json
from pathlib import Path
from urllib.request import Request, urlopen
import numpy as np
def _save_native(path, arrays, metadata):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
arrays = {k: np.asarray(v) for k, v in arrays.items()}
if any(v.dtype == object for v in arrays.values()):
raise ValueError("Native arrays must be numerical; flatten structured observations explicitly")
np.savez_compressed(path, **arrays)
metadata = {
**metadata,
"collector_version": "tfm-native-v1",
"schema_mapping_validated": False,
"normalization_status": "not fitted; keep external and synthetic sources separate",
}
path.with_suffix(".json").write_text(json.dumps(metadata, indent=2, allow_nan=False))
return metadata
def collect_citylearn(env, policy, path, *, system_id, steps=672, seed=42):
"""Collect an instantiated CityLearn 3 list-interface environment.
policy(observations, action_names, action_spaces) -> one vector per enabled
agent. Never invent thermostat controls or reinterpret storage fractions as kW.
"""
action_names = env.action_names
observation_names = env.observation_names
spaces = env.action_space
if not isinstance(spaces, list):
raise ValueError("Use CityLearn's list interface or provide a dedicated dict-interface adapter")
if any(len(names) != space.shape[0] for names, space in zip(action_names, spaces)):
raise ValueError("Native action names and action-space dimensions differ")
reset = env.reset(seed=seed)
observations = reset[0] if isinstance(reset, tuple) else reset
observed = [[np.asarray(o, dtype=float)] for o in observations]
controls = [[] for _ in spaces]
for _ in range(steps):
actions = policy(observations, action_names, spaces)
if len(actions) != len(spaces):
raise ValueError("Policy must provide every enabled agent's native actions")
validated = []
for i, (action, space) in enumerate(zip(actions, spaces)):
value = np.asarray(action, dtype=space.dtype)
if not space.contains(value) or not np.isfinite(value).all():
raise ValueError(f"Invalid native CityLearn action for agent {i}")
validated.append(value)
controls[i].append(value)
observations, _, terminated, truncated, _ = env.step(validated)
for seq, obs in zip(observed, observations):
seq.append(np.asarray(obs, dtype=float))
if terminated or truncated:
break
arrays = {
**{f"observations_{i}": x for i, x in enumerate(observed)},
**{f"actions_{i}": x for i, x in enumerate(controls)},
}
return _save_native(
path,
arrays,
{
"source": "CityLearn",
"system_id": str(system_id),
"seed": seed,
"action_names": action_names,
"observation_names": observation_names,
"seconds_per_time_step": float(env.seconds_per_time_step),
"action_bounds": [{"low": s.low.tolist(), "high": s.high.tolist()} for s in spaces],
"exogenous_status": "derive native weather/forecast separately; no future states as exogenous",
},
)
class BoptestClient:
"""Direct instance REST client. Start/select an instance outside this collector.
endpoint must already identify the instance (hosted service routes may require
their own testid prefix). Native input/measurement units are preserved.
"""
def __init__(self, endpoint, timeout=30):
self.endpoint = endpoint.rstrip("/")
self.timeout = timeout
def call(self, method, path, payload=None):
request = Request(
self.endpoint + "/" + path.lstrip("/"),
data=json.dumps(payload).encode() if payload is not None else None,
headers={"Content-Type": "application/json"},
method=method,
)
with urlopen(request, timeout=self.timeout) as response:
result = json.load(response)
if isinstance(result, dict) and "payload" in result:
if result.get("status", 200) not in (200, 201):
raise ValueError(f"BOPTEST rejected {path}: {result.get('message')}")
return result["payload"]
return result
def collect_boptest(client, policy, path, *, system_id, start_time=0, warmup_period=86400, steps=672):
"""Preserve native inputs and specified scenario/weather forecast snapshots.
policy(measurement,input_metadata) returns native overrides (including activate
flags where required). Forecast snapshots are recorded before each intervention.
"""
inputs = client.call("GET", "inputs")
measurements = client.call("GET", "measurements")
forecast_points = client.call("GET", "forecast_points")
client.call("PUT", "step", {"step": 900})
measurement = client.call("PUT", "initialize", {"start_time": start_time, "warmup_period": warmup_period})
states, actions, forecasts = [measurement], [], []
for _ in range(steps):
action = policy(measurement, inputs)
for name, value in action.items():
if name not in inputs:
raise ValueError(f"Unknown BOPTEST input {name}")
spec = inputs[name]
if (
not np.isfinite(value)
or value < spec.get("Minimum", -np.inf)
or value > spec.get("Maximum", np.inf)
):
raise ValueError(f"Invalid BOPTEST input {name}")
forecasts.append(
client.call(
"PUT", "forecast", {"point_names": list(forecast_points), "horizon": 7200, "interval": 900}
)
)
measurement = client.call("POST", "advance", action)
states.append(measurement)
actions.append(action)
# Native dictionary snapshots retain scenario-specific units and optional inputs.
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{
"source": "BOPTEST",
"collector_version": "tfm-native-v1",
"system_id": str(system_id),
"dt_seconds": 900,
"inputs": inputs,
"measurements": measurements,
"forecast_points": forecast_points,
"states": states,
"actions": actions,
"forecasts": forecasts,
"schema_mapping_validated": False,
},
indent=2,
allow_nan=False,
)
)
return target
def collect_minari(dataset, path, *, system_id_for_episode):
"""Collect numerical offline episodes; caller supplies physical-instance IDs.
An episode ID is not evidence of a distinct physical system. This is why an
explicit instance grouping function is required before any dataset split.
"""
output = Path(path)
output.mkdir(parents=True, exist_ok=True)
collected = []
for episode in dataset.iterate_episodes():
if isinstance(episode.observations, dict) or isinstance(episode.actions, dict):
raise ValueError("Structured Minari observations/actions require explicit numerical flattening")
sid = system_id_for_episode(episode)
if sid is None:
raise ValueError(
"Physical system grouping must be supplied; episode IDs cannot imply unseen systems"
)
metadata = _save_native(
output / f"episode-{episode.id}.npz",
{
"observations": episode.observations,
"actions": episode.actions,
"terminations": episode.terminations,
"truncations": episode.truncations,
},
{
"source": "Minari",
"dataset_id": dataset.spec.dataset_id,
"episode_id": int(episode.id),
"system_id": str(sid),
"intervention_coverage": "offline observed actions only",
},
)
collected.append(metadata)
(output / "native_manifest.json").write_text(json.dumps(collected, indent=2))
return collected