maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
5.27 kB
"""Full episodes and safe NPZ serialization (no pickle payloads)."""
import json
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from .schema import NUM_FIELDS, field_mask
@dataclass
class Episode:
system_id: str
family_id: str
schema: list[str]
timestamps: np.ndarray
values: np.ndarray
observation_mask: np.ndarray
actions: np.ndarray
exogenous: np.ndarray
hidden_parameters: dict
seed: int
split: str | None = None
policy_ids: np.ndarray | None = None
policy_segment_ids: np.ndarray | None = None
def __post_init__(self):
self.values = np.asarray(self.values, dtype=np.float64)
self.observation_mask = np.asarray(self.observation_mask, dtype=bool)
self.actions = np.asarray(self.actions, dtype=np.float64)
self.exogenous = np.asarray(self.exogenous, dtype=np.float64)
self.timestamps = np.asarray(self.timestamps, dtype=np.float64)
t = len(self.values) - 1
if self.values.shape != (t + 1, NUM_FIELDS) or self.observation_mask.shape != self.values.shape:
raise ValueError("Episode values/mask must be canonical [T+1,F]")
if self.actions.shape != (t, NUM_FIELDS) or self.exogenous.shape != (t, NUM_FIELDS):
raise ValueError("Episode actions/exogenous must be canonical [T,F]")
if self.timestamps.shape != (t + 1,) or np.any(np.diff(self.timestamps) <= 0):
raise ValueError("Episode timestamps must be increasing minutes")
if np.any(self.observation_mask & ~np.isfinite(self.values)):
raise ValueError("Observed episode values must be finite")
if not np.isfinite(self.actions).all() or not np.isfinite(self.exogenous).all():
raise ValueError("Applied actions and exogenous arrays must be finite")
if np.any(self.observation_mask[:, ~self.field_mask]):
raise ValueError("Padding cannot be observed")
@property
def states(self):
return self.values
@property
def field_mask(self):
return field_mask(self.schema)
def save_episode(episode, path):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
metadata = {
"system_id": episode.system_id,
"family_id": episode.family_id,
"schema": episode.schema,
"hidden_parameters": episode.hidden_parameters,
"seed": episode.seed,
"split": episode.split,
}
np.savez_compressed(
path,
metadata=np.array(json.dumps(metadata, sort_keys=True)),
timestamps=episode.timestamps,
values=episode.values,
observation_mask=episode.observation_mask,
actions=episode.actions,
exogenous=episode.exogenous,
policy_ids=episode.policy_ids if episode.policy_ids is not None else np.array([], dtype=np.int8),
policy_segment_ids=episode.policy_segment_ids
if episode.policy_segment_ids is not None
else np.array([], dtype=np.int32),
)
def load_episode(path):
with np.load(path, allow_pickle=False) as data:
metadata = json.loads(str(data["metadata"]))
return Episode(
**metadata,
timestamps=data["timestamps"],
values=data["values"],
observation_mask=data["observation_mask"],
actions=data["actions"],
exogenous=data["exogenous"],
policy_ids=data["policy_ids"],
policy_segment_ids=data["policy_segment_ids"] if "policy_segment_ids" in data else None,
)
def load_manifest_episodes(data, split=None):
path = Path(data)
manifest_path = path if path.is_file() else path / "manifest.json"
manifest = json.loads(manifest_path.read_text())
episodes = []
for entry in manifest["episodes"]:
if split is None or entry["split"] == split:
episode = load_episode(manifest_path.parent / entry["path"])
episode.split = entry["split"]
episodes.append(episode)
return episodes
def save_episode_parquet(episode, path):
"""Human-readable long rows retain masks, controls, provenance, and hidden parameters."""
import pyarrow as pa
import pyarrow.parquet as pq
from .schema import FIELD_INDEX, FIELD_SPECS
columns = {"system_id": [], "family_id": [], "time_minutes": [], "field": [], "value": [], "observed": []}
for name in episode.schema:
i = FIELD_INDEX[name]
n = len(episode.values)
columns["system_id"].extend([episode.system_id] * n)
columns["family_id"].extend([episode.family_id] * n)
columns["time_minutes"].extend(episode.timestamps.tolist())
columns["field"].extend([name] * n)
columns["value"].extend(episode.values[:, i].tolist())
columns["observed"].extend(episode.observation_mask[:, i].tolist())
table = pa.table(columns)
metadata = {
"schema": episode.schema,
"hidden_parameters": episode.hidden_parameters,
"seed": episode.seed,
"split": episode.split,
"field_specs": [vars(spec) for spec in FIELD_SPECS],
}
table = table.replace_schema_metadata({b"tfm_episode": json.dumps(metadata).encode()})
pq.write_table(table, path, compression="zstd")