File size: 5,274 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""Full episodes and safe NPZ serialization (no pickle payloads)."""

import json
from dataclasses import dataclass
from pathlib import Path

import numpy as np

from .schema import NUM_FIELDS, field_mask


@dataclass
class Episode:
    system_id: str
    family_id: str
    schema: list[str]
    timestamps: np.ndarray
    values: np.ndarray
    observation_mask: np.ndarray
    actions: np.ndarray
    exogenous: np.ndarray
    hidden_parameters: dict
    seed: int
    split: str | None = None
    policy_ids: np.ndarray | None = None
    policy_segment_ids: np.ndarray | None = None

    def __post_init__(self):
        self.values = np.asarray(self.values, dtype=np.float64)
        self.observation_mask = np.asarray(self.observation_mask, dtype=bool)
        self.actions = np.asarray(self.actions, dtype=np.float64)
        self.exogenous = np.asarray(self.exogenous, dtype=np.float64)
        self.timestamps = np.asarray(self.timestamps, dtype=np.float64)
        t = len(self.values) - 1
        if self.values.shape != (t + 1, NUM_FIELDS) or self.observation_mask.shape != self.values.shape:
            raise ValueError("Episode values/mask must be canonical [T+1,F]")
        if self.actions.shape != (t, NUM_FIELDS) or self.exogenous.shape != (t, NUM_FIELDS):
            raise ValueError("Episode actions/exogenous must be canonical [T,F]")
        if self.timestamps.shape != (t + 1,) or np.any(np.diff(self.timestamps) <= 0):
            raise ValueError("Episode timestamps must be increasing minutes")
        if np.any(self.observation_mask & ~np.isfinite(self.values)):
            raise ValueError("Observed episode values must be finite")
        if not np.isfinite(self.actions).all() or not np.isfinite(self.exogenous).all():
            raise ValueError("Applied actions and exogenous arrays must be finite")
        if np.any(self.observation_mask[:, ~self.field_mask]):
            raise ValueError("Padding cannot be observed")

    @property
    def states(self):
        return self.values

    @property
    def field_mask(self):
        return field_mask(self.schema)


def save_episode(episode, path):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    metadata = {
        "system_id": episode.system_id,
        "family_id": episode.family_id,
        "schema": episode.schema,
        "hidden_parameters": episode.hidden_parameters,
        "seed": episode.seed,
        "split": episode.split,
    }
    np.savez_compressed(
        path,
        metadata=np.array(json.dumps(metadata, sort_keys=True)),
        timestamps=episode.timestamps,
        values=episode.values,
        observation_mask=episode.observation_mask,
        actions=episode.actions,
        exogenous=episode.exogenous,
        policy_ids=episode.policy_ids if episode.policy_ids is not None else np.array([], dtype=np.int8),
        policy_segment_ids=episode.policy_segment_ids
        if episode.policy_segment_ids is not None
        else np.array([], dtype=np.int32),
    )


def load_episode(path):
    with np.load(path, allow_pickle=False) as data:
        metadata = json.loads(str(data["metadata"]))
        return Episode(
            **metadata,
            timestamps=data["timestamps"],
            values=data["values"],
            observation_mask=data["observation_mask"],
            actions=data["actions"],
            exogenous=data["exogenous"],
            policy_ids=data["policy_ids"],
            policy_segment_ids=data["policy_segment_ids"] if "policy_segment_ids" in data else None,
        )


def load_manifest_episodes(data, split=None):
    path = Path(data)
    manifest_path = path if path.is_file() else path / "manifest.json"
    manifest = json.loads(manifest_path.read_text())
    episodes = []
    for entry in manifest["episodes"]:
        if split is None or entry["split"] == split:
            episode = load_episode(manifest_path.parent / entry["path"])
            episode.split = entry["split"]
            episodes.append(episode)
    return episodes


def save_episode_parquet(episode, path):
    """Human-readable long rows retain masks, controls, provenance, and hidden parameters."""
    import pyarrow as pa
    import pyarrow.parquet as pq

    from .schema import FIELD_INDEX, FIELD_SPECS

    columns = {"system_id": [], "family_id": [], "time_minutes": [], "field": [], "value": [], "observed": []}
    for name in episode.schema:
        i = FIELD_INDEX[name]
        n = len(episode.values)
        columns["system_id"].extend([episode.system_id] * n)
        columns["family_id"].extend([episode.family_id] * n)
        columns["time_minutes"].extend(episode.timestamps.tolist())
        columns["field"].extend([name] * n)
        columns["value"].extend(episode.values[:, i].tolist())
        columns["observed"].extend(episode.observation_mask[:, i].tolist())
    table = pa.table(columns)
    metadata = {
        "schema": episode.schema,
        "hidden_parameters": episode.hidden_parameters,
        "seed": episode.seed,
        "split": episode.split,
        "field_specs": [vars(spec) for spec in FIELD_SPECS],
    }
    table = table.replace_schema_metadata({b"tfm_episode": json.dumps(metadata).encode()})
    pq.write_table(table, path, compression="zstd")