File size: 4,616 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Explicit experiment provenance and observation-only stress transformations."""

from __future__ import annotations

import json
from dataclasses import asdict, dataclass
from pathlib import Path

import numpy as np
import torch


@dataclass
class ExperimentConfig:
    name: str
    axis: str = "B"
    data: str | None = None
    checkpoint: str | None = None
    excluded_family: str | None = None
    sensor_failure_probability: float = 0.0
    sensor_bias_std: float = 0.0
    description: str = ""

    def validate(self):
        if self.axis not in ("B", "C", "D", "E"):
            raise ValueError("Experiment axis must be B, C, D or E")
        if not 0 <= self.sensor_failure_probability <= 1 or self.sensor_bias_std < 0:
            raise ValueError("Invalid sensor stress settings")
        if self.axis == "E" and not self.excluded_family:
            raise ValueError("Leave-family-out evaluation needs an excluded family")
        return self


def default_experiments():
    return [
        ExperimentConfig("heldout_systems", "B", description="Main held-out-system test"),
        ExperimentConfig(
            "sensor_failures",
            "C",
            sensor_failure_probability=0.25,
            description="Observed history/current sensor dropout",
        ),
        ExperimentConfig(
            "sensor_bias",
            "C",
            sensor_bias_std=0.5,
            description="Observed state sensor bias in training-standardized units",
        ),
        ExperimentConfig(
            "configuration_shift",
            "D",
            description="Requires independently generated shifted physical parameters",
        ),
        *[
            ExperimentConfig(
                f"leave_{family}_out",
                "E",
                excluded_family=family,
                description="Requires checkpoint trained with this family excluded",
            )
            for family in ("thermal", "tank", "battery")
        ],
    ]


def stress_observations(batch, config, seed=0):
    """Corrupt state observations, never targets or supplied future controls."""
    config.validate()
    out = {k: v.clone() if torch.is_tensor(v) else v for k, v in batch.items()}
    rng = np.random.default_rng(seed)
    state = out["role_ids"] == 0
    if config.sensor_failure_probability:
        dropped = (
            torch.tensor(rng.random(state.shape) < config.sensor_failure_probability, device=state.device)
            & state
        )
        out["history_mask"] = out["history_mask"] * (~dropped[:, None, :])
        out["current_mask"] = out["current_mask"] * (~dropped)
    if config.sensor_bias_std:
        bias = (
            torch.tensor(
                rng.normal(0, config.sensor_bias_std, state.shape),
                dtype=out["history"].dtype,
                device=state.device,
            )
            * state
        )
        out["history"] = out["history"] + bias[:, None, :]
        out["current"] = out["current"] + bias
    out["history"] *= out["history_mask"]
    out["current"] *= out["current_mask"]
    return out


def run_experiments(configs, evaluator, output):
    """Evaluator receives validated config; absent prerequisites remain unrun."""
    results = []
    for config in configs:
        config.validate()
        row = {"config": asdict(config), "status": "unrun"}
        if not config.data or not config.checkpoint:
            row["reason"] = "Dataset and compatible trained checkpoint were not supplied"
        elif config.axis == "E":
            run_path = Path(config.checkpoint).parent / "run.json"
            run = json.loads(run_path.read_text()) if run_path.exists() else {}
            if config.excluded_family not in run.get("excluded_families", []):
                row["reason"] = "Checkpoint has no provenance proving the family was excluded from training"
            else:
                row.update(status="completed", result=evaluator(config))
        elif config.axis == "D":
            manifest_path = Path(config.data) / "manifest.json"
            manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
            if not manifest.get("configuration_shift"):
                row["reason"] = "Dataset manifest has no declared configuration shift"
            else:
                row.update(status="completed", result=evaluator(config))
        else:
            row.update(status="completed", result=evaluator(config))
        results.append(row)
    Path(output).write_text(json.dumps({"experiments": results}, indent=2, allow_nan=False) + "\n")
    return results