"""Isolated native collectors. Raw external names/units require a validated mapping before training. These collectors deliberately never claim arbitrary external observations are the synthetic vocabulary. They preserve native controls and provenance for milestone 5. """ from __future__ import annotations import json from pathlib import Path from urllib.request import Request, urlopen import numpy as np def _save_native(path, arrays, metadata): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) arrays = {k: np.asarray(v) for k, v in arrays.items()} if any(v.dtype == object for v in arrays.values()): raise ValueError("Native arrays must be numerical; flatten structured observations explicitly") np.savez_compressed(path, **arrays) metadata = { **metadata, "collector_version": "tfm-native-v1", "schema_mapping_validated": False, "normalization_status": "not fitted; keep external and synthetic sources separate", } path.with_suffix(".json").write_text(json.dumps(metadata, indent=2, allow_nan=False)) return metadata def collect_citylearn(env, policy, path, *, system_id, steps=672, seed=42): """Collect an instantiated CityLearn 3 list-interface environment. policy(observations, action_names, action_spaces) -> one vector per enabled agent. Never invent thermostat controls or reinterpret storage fractions as kW. """ action_names = env.action_names observation_names = env.observation_names spaces = env.action_space if not isinstance(spaces, list): raise ValueError("Use CityLearn's list interface or provide a dedicated dict-interface adapter") if any(len(names) != space.shape[0] for names, space in zip(action_names, spaces)): raise ValueError("Native action names and action-space dimensions differ") reset = env.reset(seed=seed) observations = reset[0] if isinstance(reset, tuple) else reset observed = [[np.asarray(o, dtype=float)] for o in observations] controls = [[] for _ in spaces] for _ in range(steps): actions = policy(observations, action_names, spaces) if len(actions) != len(spaces): raise ValueError("Policy must provide every enabled agent's native actions") validated = [] for i, (action, space) in enumerate(zip(actions, spaces)): value = np.asarray(action, dtype=space.dtype) if not space.contains(value) or not np.isfinite(value).all(): raise ValueError(f"Invalid native CityLearn action for agent {i}") validated.append(value) controls[i].append(value) observations, _, terminated, truncated, _ = env.step(validated) for seq, obs in zip(observed, observations): seq.append(np.asarray(obs, dtype=float)) if terminated or truncated: break arrays = { **{f"observations_{i}": x for i, x in enumerate(observed)}, **{f"actions_{i}": x for i, x in enumerate(controls)}, } return _save_native( path, arrays, { "source": "CityLearn", "system_id": str(system_id), "seed": seed, "action_names": action_names, "observation_names": observation_names, "seconds_per_time_step": float(env.seconds_per_time_step), "action_bounds": [{"low": s.low.tolist(), "high": s.high.tolist()} for s in spaces], "exogenous_status": "derive native weather/forecast separately; no future states as exogenous", }, ) class BoptestClient: """Direct instance REST client. Start/select an instance outside this collector. endpoint must already identify the instance (hosted service routes may require their own testid prefix). Native input/measurement units are preserved. """ def __init__(self, endpoint, timeout=30): self.endpoint = endpoint.rstrip("/") self.timeout = timeout def call(self, method, path, payload=None): request = Request( self.endpoint + "/" + path.lstrip("/"), data=json.dumps(payload).encode() if payload is not None else None, headers={"Content-Type": "application/json"}, method=method, ) with urlopen(request, timeout=self.timeout) as response: result = json.load(response) if isinstance(result, dict) and "payload" in result: if result.get("status", 200) not in (200, 201): raise ValueError(f"BOPTEST rejected {path}: {result.get('message')}") return result["payload"] return result def collect_boptest(client, policy, path, *, system_id, start_time=0, warmup_period=86400, steps=672): """Preserve native inputs and specified scenario/weather forecast snapshots. policy(measurement,input_metadata) returns native overrides (including activate flags where required). Forecast snapshots are recorded before each intervention. """ inputs = client.call("GET", "inputs") measurements = client.call("GET", "measurements") forecast_points = client.call("GET", "forecast_points") client.call("PUT", "step", {"step": 900}) measurement = client.call("PUT", "initialize", {"start_time": start_time, "warmup_period": warmup_period}) states, actions, forecasts = [measurement], [], [] for _ in range(steps): action = policy(measurement, inputs) for name, value in action.items(): if name not in inputs: raise ValueError(f"Unknown BOPTEST input {name}") spec = inputs[name] if ( not np.isfinite(value) or value < spec.get("Minimum", -np.inf) or value > spec.get("Maximum", np.inf) ): raise ValueError(f"Invalid BOPTEST input {name}") forecasts.append( client.call( "PUT", "forecast", {"point_names": list(forecast_points), "horizon": 7200, "interval": 900} ) ) measurement = client.call("POST", "advance", action) states.append(measurement) actions.append(action) # Native dictionary snapshots retain scenario-specific units and optional inputs. target = Path(path) target.parent.mkdir(parents=True, exist_ok=True) target.write_text( json.dumps( { "source": "BOPTEST", "collector_version": "tfm-native-v1", "system_id": str(system_id), "dt_seconds": 900, "inputs": inputs, "measurements": measurements, "forecast_points": forecast_points, "states": states, "actions": actions, "forecasts": forecasts, "schema_mapping_validated": False, }, indent=2, allow_nan=False, ) ) return target def collect_minari(dataset, path, *, system_id_for_episode): """Collect numerical offline episodes; caller supplies physical-instance IDs. An episode ID is not evidence of a distinct physical system. This is why an explicit instance grouping function is required before any dataset split. """ output = Path(path) output.mkdir(parents=True, exist_ok=True) collected = [] for episode in dataset.iterate_episodes(): if isinstance(episode.observations, dict) or isinstance(episode.actions, dict): raise ValueError("Structured Minari observations/actions require explicit numerical flattening") sid = system_id_for_episode(episode) if sid is None: raise ValueError( "Physical system grouping must be supplied; episode IDs cannot imply unseen systems" ) metadata = _save_native( output / f"episode-{episode.id}.npz", { "observations": episode.observations, "actions": episode.actions, "terminations": episode.terminations, "truncations": episode.truncations, }, { "source": "Minari", "dataset_id": dataset.spec.dataset_id, "episode_id": int(episode.id), "system_id": str(sid), "intervention_coverage": "offline observed actions only", }, ) collected.append(metadata) (output / "native_manifest.json").write_text(json.dumps(collected, indent=2)) return collected