File size: 8,604 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
"""Isolated native collectors. Raw external names/units require a validated mapping before training.

These collectors deliberately never claim arbitrary external observations are the
synthetic vocabulary. They preserve native controls and provenance for milestone 5.
"""

from __future__ import annotations

import json
from pathlib import Path
from urllib.request import Request, urlopen

import numpy as np


def _save_native(path, arrays, metadata):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    arrays = {k: np.asarray(v) for k, v in arrays.items()}
    if any(v.dtype == object for v in arrays.values()):
        raise ValueError("Native arrays must be numerical; flatten structured observations explicitly")
    np.savez_compressed(path, **arrays)
    metadata = {
        **metadata,
        "collector_version": "tfm-native-v1",
        "schema_mapping_validated": False,
        "normalization_status": "not fitted; keep external and synthetic sources separate",
    }
    path.with_suffix(".json").write_text(json.dumps(metadata, indent=2, allow_nan=False))
    return metadata


def collect_citylearn(env, policy, path, *, system_id, steps=672, seed=42):
    """Collect an instantiated CityLearn 3 list-interface environment.

    policy(observations, action_names, action_spaces) -> one vector per enabled
    agent. Never invent thermostat controls or reinterpret storage fractions as kW.
    """
    action_names = env.action_names
    observation_names = env.observation_names
    spaces = env.action_space
    if not isinstance(spaces, list):
        raise ValueError("Use CityLearn's list interface or provide a dedicated dict-interface adapter")
    if any(len(names) != space.shape[0] for names, space in zip(action_names, spaces)):
        raise ValueError("Native action names and action-space dimensions differ")
    reset = env.reset(seed=seed)
    observations = reset[0] if isinstance(reset, tuple) else reset
    observed = [[np.asarray(o, dtype=float)] for o in observations]
    controls = [[] for _ in spaces]
    for _ in range(steps):
        actions = policy(observations, action_names, spaces)
        if len(actions) != len(spaces):
            raise ValueError("Policy must provide every enabled agent's native actions")
        validated = []
        for i, (action, space) in enumerate(zip(actions, spaces)):
            value = np.asarray(action, dtype=space.dtype)
            if not space.contains(value) or not np.isfinite(value).all():
                raise ValueError(f"Invalid native CityLearn action for agent {i}")
            validated.append(value)
            controls[i].append(value)
        observations, _, terminated, truncated, _ = env.step(validated)
        for seq, obs in zip(observed, observations):
            seq.append(np.asarray(obs, dtype=float))
        if terminated or truncated:
            break
    arrays = {
        **{f"observations_{i}": x for i, x in enumerate(observed)},
        **{f"actions_{i}": x for i, x in enumerate(controls)},
    }
    return _save_native(
        path,
        arrays,
        {
            "source": "CityLearn",
            "system_id": str(system_id),
            "seed": seed,
            "action_names": action_names,
            "observation_names": observation_names,
            "seconds_per_time_step": float(env.seconds_per_time_step),
            "action_bounds": [{"low": s.low.tolist(), "high": s.high.tolist()} for s in spaces],
            "exogenous_status": "derive native weather/forecast separately; no future states as exogenous",
        },
    )


class BoptestClient:
    """Direct instance REST client. Start/select an instance outside this collector.

    endpoint must already identify the instance (hosted service routes may require
    their own testid prefix). Native input/measurement units are preserved.
    """

    def __init__(self, endpoint, timeout=30):
        self.endpoint = endpoint.rstrip("/")
        self.timeout = timeout

    def call(self, method, path, payload=None):
        request = Request(
            self.endpoint + "/" + path.lstrip("/"),
            data=json.dumps(payload).encode() if payload is not None else None,
            headers={"Content-Type": "application/json"},
            method=method,
        )
        with urlopen(request, timeout=self.timeout) as response:
            result = json.load(response)
        if isinstance(result, dict) and "payload" in result:
            if result.get("status", 200) not in (200, 201):
                raise ValueError(f"BOPTEST rejected {path}: {result.get('message')}")
            return result["payload"]
        return result


def collect_boptest(client, policy, path, *, system_id, start_time=0, warmup_period=86400, steps=672):
    """Preserve native inputs and specified scenario/weather forecast snapshots.

    policy(measurement,input_metadata) returns native overrides (including activate
    flags where required). Forecast snapshots are recorded before each intervention.
    """
    inputs = client.call("GET", "inputs")
    measurements = client.call("GET", "measurements")
    forecast_points = client.call("GET", "forecast_points")
    client.call("PUT", "step", {"step": 900})
    measurement = client.call("PUT", "initialize", {"start_time": start_time, "warmup_period": warmup_period})
    states, actions, forecasts = [measurement], [], []
    for _ in range(steps):
        action = policy(measurement, inputs)
        for name, value in action.items():
            if name not in inputs:
                raise ValueError(f"Unknown BOPTEST input {name}")
            spec = inputs[name]
            if (
                not np.isfinite(value)
                or value < spec.get("Minimum", -np.inf)
                or value > spec.get("Maximum", np.inf)
            ):
                raise ValueError(f"Invalid BOPTEST input {name}")
        forecasts.append(
            client.call(
                "PUT", "forecast", {"point_names": list(forecast_points), "horizon": 7200, "interval": 900}
            )
        )
        measurement = client.call("POST", "advance", action)
        states.append(measurement)
        actions.append(action)
    # Native dictionary snapshots retain scenario-specific units and optional inputs.
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_text(
        json.dumps(
            {
                "source": "BOPTEST",
                "collector_version": "tfm-native-v1",
                "system_id": str(system_id),
                "dt_seconds": 900,
                "inputs": inputs,
                "measurements": measurements,
                "forecast_points": forecast_points,
                "states": states,
                "actions": actions,
                "forecasts": forecasts,
                "schema_mapping_validated": False,
            },
            indent=2,
            allow_nan=False,
        )
    )
    return target


def collect_minari(dataset, path, *, system_id_for_episode):
    """Collect numerical offline episodes; caller supplies physical-instance IDs.

    An episode ID is not evidence of a distinct physical system. This is why an
    explicit instance grouping function is required before any dataset split.
    """
    output = Path(path)
    output.mkdir(parents=True, exist_ok=True)
    collected = []
    for episode in dataset.iterate_episodes():
        if isinstance(episode.observations, dict) or isinstance(episode.actions, dict):
            raise ValueError("Structured Minari observations/actions require explicit numerical flattening")
        sid = system_id_for_episode(episode)
        if sid is None:
            raise ValueError(
                "Physical system grouping must be supplied; episode IDs cannot imply unseen systems"
            )
        metadata = _save_native(
            output / f"episode-{episode.id}.npz",
            {
                "observations": episode.observations,
                "actions": episode.actions,
                "terminations": episode.terminations,
                "truncations": episode.truncations,
            },
            {
                "source": "Minari",
                "dataset_id": dataset.spec.dataset_id,
                "episode_id": int(episode.id),
                "system_id": str(sid),
                "intervention_coverage": "offline observed actions only",
            },
        )
        collected.append(metadata)
    (output / "native_manifest.json").write_text(json.dumps(collected, indent=2))
    return collected