Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/data/episodes.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 5.27 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/episodes.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/data/episodes.py
-
curl -L -o episodes.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/data/episodes.py
5.27 kB
| """Full episodes and safe NPZ serialization (no pickle payloads).""" | |
| import json | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import numpy as np | |
| from .schema import NUM_FIELDS, field_mask | |
| class Episode: | |
| system_id: str | |
| family_id: str | |
| schema: list[str] | |
| timestamps: np.ndarray | |
| values: np.ndarray | |
| observation_mask: np.ndarray | |
| actions: np.ndarray | |
| exogenous: np.ndarray | |
| hidden_parameters: dict | |
| seed: int | |
| split: str | None = None | |
| policy_ids: np.ndarray | None = None | |
| policy_segment_ids: np.ndarray | None = None | |
| def __post_init__(self): | |
| self.values = np.asarray(self.values, dtype=np.float64) | |
| self.observation_mask = np.asarray(self.observation_mask, dtype=bool) | |
| self.actions = np.asarray(self.actions, dtype=np.float64) | |
| self.exogenous = np.asarray(self.exogenous, dtype=np.float64) | |
| self.timestamps = np.asarray(self.timestamps, dtype=np.float64) | |
| t = len(self.values) - 1 | |
| if self.values.shape != (t + 1, NUM_FIELDS) or self.observation_mask.shape != self.values.shape: | |
| raise ValueError("Episode values/mask must be canonical [T+1,F]") | |
| if self.actions.shape != (t, NUM_FIELDS) or self.exogenous.shape != (t, NUM_FIELDS): | |
| raise ValueError("Episode actions/exogenous must be canonical [T,F]") | |
| if self.timestamps.shape != (t + 1,) or np.any(np.diff(self.timestamps) <= 0): | |
| raise ValueError("Episode timestamps must be increasing minutes") | |
| if np.any(self.observation_mask & ~np.isfinite(self.values)): | |
| raise ValueError("Observed episode values must be finite") | |
| if not np.isfinite(self.actions).all() or not np.isfinite(self.exogenous).all(): | |
| raise ValueError("Applied actions and exogenous arrays must be finite") | |
| if np.any(self.observation_mask[:, ~self.field_mask]): | |
| raise ValueError("Padding cannot be observed") | |
| def states(self): | |
| return self.values | |
| def field_mask(self): | |
| return field_mask(self.schema) | |
| def save_episode(episode, path): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| metadata = { | |
| "system_id": episode.system_id, | |
| "family_id": episode.family_id, | |
| "schema": episode.schema, | |
| "hidden_parameters": episode.hidden_parameters, | |
| "seed": episode.seed, | |
| "split": episode.split, | |
| } | |
| np.savez_compressed( | |
| path, | |
| metadata=np.array(json.dumps(metadata, sort_keys=True)), | |
| timestamps=episode.timestamps, | |
| values=episode.values, | |
| observation_mask=episode.observation_mask, | |
| actions=episode.actions, | |
| exogenous=episode.exogenous, | |
| policy_ids=episode.policy_ids if episode.policy_ids is not None else np.array([], dtype=np.int8), | |
| policy_segment_ids=episode.policy_segment_ids | |
| if episode.policy_segment_ids is not None | |
| else np.array([], dtype=np.int32), | |
| ) | |
| def load_episode(path): | |
| with np.load(path, allow_pickle=False) as data: | |
| metadata = json.loads(str(data["metadata"])) | |
| return Episode( | |
| **metadata, | |
| timestamps=data["timestamps"], | |
| values=data["values"], | |
| observation_mask=data["observation_mask"], | |
| actions=data["actions"], | |
| exogenous=data["exogenous"], | |
| policy_ids=data["policy_ids"], | |
| policy_segment_ids=data["policy_segment_ids"] if "policy_segment_ids" in data else None, | |
| ) | |
| def load_manifest_episodes(data, split=None): | |
| path = Path(data) | |
| manifest_path = path if path.is_file() else path / "manifest.json" | |
| manifest = json.loads(manifest_path.read_text()) | |
| episodes = [] | |
| for entry in manifest["episodes"]: | |
| if split is None or entry["split"] == split: | |
| episode = load_episode(manifest_path.parent / entry["path"]) | |
| episode.split = entry["split"] | |
| episodes.append(episode) | |
| return episodes | |
| def save_episode_parquet(episode, path): | |
| """Human-readable long rows retain masks, controls, provenance, and hidden parameters.""" | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| from .schema import FIELD_INDEX, FIELD_SPECS | |
| columns = {"system_id": [], "family_id": [], "time_minutes": [], "field": [], "value": [], "observed": []} | |
| for name in episode.schema: | |
| i = FIELD_INDEX[name] | |
| n = len(episode.values) | |
| columns["system_id"].extend([episode.system_id] * n) | |
| columns["family_id"].extend([episode.family_id] * n) | |
| columns["time_minutes"].extend(episode.timestamps.tolist()) | |
| columns["field"].extend([name] * n) | |
| columns["value"].extend(episode.values[:, i].tolist()) | |
| columns["observed"].extend(episode.observation_mask[:, i].tolist()) | |
| table = pa.table(columns) | |
| metadata = { | |
| "schema": episode.schema, | |
| "hidden_parameters": episode.hidden_parameters, | |
| "seed": episode.seed, | |
| "split": episode.split, | |
| "field_specs": [vars(spec) for spec in FIELD_SPECS], | |
| } | |
| table = table.replace_schema_metadata({b"tfm_episode": json.dumps(metadata).encode()}) | |
| pq.write_table(table, path, compression="zstd") | |