File size: 5,274 Bytes
1baf215 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Full episodes and safe NPZ serialization (no pickle payloads)."""
import json
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from .schema import NUM_FIELDS, field_mask
@dataclass
class Episode:
system_id: str
family_id: str
schema: list[str]
timestamps: np.ndarray
values: np.ndarray
observation_mask: np.ndarray
actions: np.ndarray
exogenous: np.ndarray
hidden_parameters: dict
seed: int
split: str | None = None
policy_ids: np.ndarray | None = None
policy_segment_ids: np.ndarray | None = None
def __post_init__(self):
self.values = np.asarray(self.values, dtype=np.float64)
self.observation_mask = np.asarray(self.observation_mask, dtype=bool)
self.actions = np.asarray(self.actions, dtype=np.float64)
self.exogenous = np.asarray(self.exogenous, dtype=np.float64)
self.timestamps = np.asarray(self.timestamps, dtype=np.float64)
t = len(self.values) - 1
if self.values.shape != (t + 1, NUM_FIELDS) or self.observation_mask.shape != self.values.shape:
raise ValueError("Episode values/mask must be canonical [T+1,F]")
if self.actions.shape != (t, NUM_FIELDS) or self.exogenous.shape != (t, NUM_FIELDS):
raise ValueError("Episode actions/exogenous must be canonical [T,F]")
if self.timestamps.shape != (t + 1,) or np.any(np.diff(self.timestamps) <= 0):
raise ValueError("Episode timestamps must be increasing minutes")
if np.any(self.observation_mask & ~np.isfinite(self.values)):
raise ValueError("Observed episode values must be finite")
if not np.isfinite(self.actions).all() or not np.isfinite(self.exogenous).all():
raise ValueError("Applied actions and exogenous arrays must be finite")
if np.any(self.observation_mask[:, ~self.field_mask]):
raise ValueError("Padding cannot be observed")
@property
def states(self):
return self.values
@property
def field_mask(self):
return field_mask(self.schema)
def save_episode(episode, path):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
metadata = {
"system_id": episode.system_id,
"family_id": episode.family_id,
"schema": episode.schema,
"hidden_parameters": episode.hidden_parameters,
"seed": episode.seed,
"split": episode.split,
}
np.savez_compressed(
path,
metadata=np.array(json.dumps(metadata, sort_keys=True)),
timestamps=episode.timestamps,
values=episode.values,
observation_mask=episode.observation_mask,
actions=episode.actions,
exogenous=episode.exogenous,
policy_ids=episode.policy_ids if episode.policy_ids is not None else np.array([], dtype=np.int8),
policy_segment_ids=episode.policy_segment_ids
if episode.policy_segment_ids is not None
else np.array([], dtype=np.int32),
)
def load_episode(path):
with np.load(path, allow_pickle=False) as data:
metadata = json.loads(str(data["metadata"]))
return Episode(
**metadata,
timestamps=data["timestamps"],
values=data["values"],
observation_mask=data["observation_mask"],
actions=data["actions"],
exogenous=data["exogenous"],
policy_ids=data["policy_ids"],
policy_segment_ids=data["policy_segment_ids"] if "policy_segment_ids" in data else None,
)
def load_manifest_episodes(data, split=None):
path = Path(data)
manifest_path = path if path.is_file() else path / "manifest.json"
manifest = json.loads(manifest_path.read_text())
episodes = []
for entry in manifest["episodes"]:
if split is None or entry["split"] == split:
episode = load_episode(manifest_path.parent / entry["path"])
episode.split = entry["split"]
episodes.append(episode)
return episodes
def save_episode_parquet(episode, path):
"""Human-readable long rows retain masks, controls, provenance, and hidden parameters."""
import pyarrow as pa
import pyarrow.parquet as pq
from .schema import FIELD_INDEX, FIELD_SPECS
columns = {"system_id": [], "family_id": [], "time_minutes": [], "field": [], "value": [], "observed": []}
for name in episode.schema:
i = FIELD_INDEX[name]
n = len(episode.values)
columns["system_id"].extend([episode.system_id] * n)
columns["family_id"].extend([episode.family_id] * n)
columns["time_minutes"].extend(episode.timestamps.tolist())
columns["field"].extend([name] * n)
columns["value"].extend(episode.values[:, i].tolist())
columns["observed"].extend(episode.observation_mask[:, i].tolist())
table = pa.table(columns)
metadata = {
"schema": episode.schema,
"hidden_parameters": episode.hidden_parameters,
"seed": episode.seed,
"split": episode.split,
"field_specs": [vars(spec) for spec in FIELD_SPECS],
}
table = table.replace_schema_metadata({b"tfm_episode": json.dumps(metadata).encode()})
pq.write_table(table, path, compression="zstd")
|