"""Full episodes and safe NPZ serialization (no pickle payloads).""" import json from dataclasses import dataclass from pathlib import Path import numpy as np from .schema import NUM_FIELDS, field_mask @dataclass class Episode: system_id: str family_id: str schema: list[str] timestamps: np.ndarray values: np.ndarray observation_mask: np.ndarray actions: np.ndarray exogenous: np.ndarray hidden_parameters: dict seed: int split: str | None = None policy_ids: np.ndarray | None = None policy_segment_ids: np.ndarray | None = None def __post_init__(self): self.values = np.asarray(self.values, dtype=np.float64) self.observation_mask = np.asarray(self.observation_mask, dtype=bool) self.actions = np.asarray(self.actions, dtype=np.float64) self.exogenous = np.asarray(self.exogenous, dtype=np.float64) self.timestamps = np.asarray(self.timestamps, dtype=np.float64) t = len(self.values) - 1 if self.values.shape != (t + 1, NUM_FIELDS) or self.observation_mask.shape != self.values.shape: raise ValueError("Episode values/mask must be canonical [T+1,F]") if self.actions.shape != (t, NUM_FIELDS) or self.exogenous.shape != (t, NUM_FIELDS): raise ValueError("Episode actions/exogenous must be canonical [T,F]") if self.timestamps.shape != (t + 1,) or np.any(np.diff(self.timestamps) <= 0): raise ValueError("Episode timestamps must be increasing minutes") if np.any(self.observation_mask & ~np.isfinite(self.values)): raise ValueError("Observed episode values must be finite") if not np.isfinite(self.actions).all() or not np.isfinite(self.exogenous).all(): raise ValueError("Applied actions and exogenous arrays must be finite") if np.any(self.observation_mask[:, ~self.field_mask]): raise ValueError("Padding cannot be observed") @property def states(self): return self.values @property def field_mask(self): return field_mask(self.schema) def save_episode(episode, path): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) metadata = { "system_id": episode.system_id, "family_id": episode.family_id, "schema": episode.schema, "hidden_parameters": episode.hidden_parameters, "seed": episode.seed, "split": episode.split, } np.savez_compressed( path, metadata=np.array(json.dumps(metadata, sort_keys=True)), timestamps=episode.timestamps, values=episode.values, observation_mask=episode.observation_mask, actions=episode.actions, exogenous=episode.exogenous, policy_ids=episode.policy_ids if episode.policy_ids is not None else np.array([], dtype=np.int8), policy_segment_ids=episode.policy_segment_ids if episode.policy_segment_ids is not None else np.array([], dtype=np.int32), ) def load_episode(path): with np.load(path, allow_pickle=False) as data: metadata = json.loads(str(data["metadata"])) return Episode( **metadata, timestamps=data["timestamps"], values=data["values"], observation_mask=data["observation_mask"], actions=data["actions"], exogenous=data["exogenous"], policy_ids=data["policy_ids"], policy_segment_ids=data["policy_segment_ids"] if "policy_segment_ids" in data else None, ) def load_manifest_episodes(data, split=None): path = Path(data) manifest_path = path if path.is_file() else path / "manifest.json" manifest = json.loads(manifest_path.read_text()) episodes = [] for entry in manifest["episodes"]: if split is None or entry["split"] == split: episode = load_episode(manifest_path.parent / entry["path"]) episode.split = entry["split"] episodes.append(episode) return episodes def save_episode_parquet(episode, path): """Human-readable long rows retain masks, controls, provenance, and hidden parameters.""" import pyarrow as pa import pyarrow.parquet as pq from .schema import FIELD_INDEX, FIELD_SPECS columns = {"system_id": [], "family_id": [], "time_minutes": [], "field": [], "value": [], "observed": []} for name in episode.schema: i = FIELD_INDEX[name] n = len(episode.values) columns["system_id"].extend([episode.system_id] * n) columns["family_id"].extend([episode.family_id] * n) columns["time_minutes"].extend(episode.timestamps.tolist()) columns["field"].extend([name] * n) columns["value"].extend(episode.values[:, i].tolist()) columns["observed"].extend(episode.observation_mask[:, i].tolist()) table = pa.table(columns) metadata = { "schema": episode.schema, "hidden_parameters": episode.hidden_parameters, "seed": episode.seed, "split": episode.split, "field_specs": [vars(spec) for spec in FIELD_SPECS], } table = table.replace_schema_metadata({b"tfm_episode": json.dumps(metadata).encode()}) pq.write_table(table, path, compression="zstd")