maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
4.62 kB
"""Explicit experiment provenance and observation-only stress transformations."""
from __future__ import annotations
import json
from dataclasses import asdict, dataclass
from pathlib import Path
import numpy as np
import torch
@dataclass
class ExperimentConfig:
name: str
axis: str = "B"
data: str | None = None
checkpoint: str | None = None
excluded_family: str | None = None
sensor_failure_probability: float = 0.0
sensor_bias_std: float = 0.0
description: str = ""
def validate(self):
if self.axis not in ("B", "C", "D", "E"):
raise ValueError("Experiment axis must be B, C, D or E")
if not 0 <= self.sensor_failure_probability <= 1 or self.sensor_bias_std < 0:
raise ValueError("Invalid sensor stress settings")
if self.axis == "E" and not self.excluded_family:
raise ValueError("Leave-family-out evaluation needs an excluded family")
return self
def default_experiments():
return [
ExperimentConfig("heldout_systems", "B", description="Main held-out-system test"),
ExperimentConfig(
"sensor_failures",
"C",
sensor_failure_probability=0.25,
description="Observed history/current sensor dropout",
),
ExperimentConfig(
"sensor_bias",
"C",
sensor_bias_std=0.5,
description="Observed state sensor bias in training-standardized units",
),
ExperimentConfig(
"configuration_shift",
"D",
description="Requires independently generated shifted physical parameters",
),
*[
ExperimentConfig(
f"leave_{family}_out",
"E",
excluded_family=family,
description="Requires checkpoint trained with this family excluded",
)
for family in ("thermal", "tank", "battery")
],
]
def stress_observations(batch, config, seed=0):
"""Corrupt state observations, never targets or supplied future controls."""
config.validate()
out = {k: v.clone() if torch.is_tensor(v) else v for k, v in batch.items()}
rng = np.random.default_rng(seed)
state = out["role_ids"] == 0
if config.sensor_failure_probability:
dropped = (
torch.tensor(rng.random(state.shape) < config.sensor_failure_probability, device=state.device)
& state
)
out["history_mask"] = out["history_mask"] * (~dropped[:, None, :])
out["current_mask"] = out["current_mask"] * (~dropped)
if config.sensor_bias_std:
bias = (
torch.tensor(
rng.normal(0, config.sensor_bias_std, state.shape),
dtype=out["history"].dtype,
device=state.device,
)
* state
)
out["history"] = out["history"] + bias[:, None, :]
out["current"] = out["current"] + bias
out["history"] *= out["history_mask"]
out["current"] *= out["current_mask"]
return out
def run_experiments(configs, evaluator, output):
"""Evaluator receives validated config; absent prerequisites remain unrun."""
results = []
for config in configs:
config.validate()
row = {"config": asdict(config), "status": "unrun"}
if not config.data or not config.checkpoint:
row["reason"] = "Dataset and compatible trained checkpoint were not supplied"
elif config.axis == "E":
run_path = Path(config.checkpoint).parent / "run.json"
run = json.loads(run_path.read_text()) if run_path.exists() else {}
if config.excluded_family not in run.get("excluded_families", []):
row["reason"] = "Checkpoint has no provenance proving the family was excluded from training"
else:
row.update(status="completed", result=evaluator(config))
elif config.axis == "D":
manifest_path = Path(config.data) / "manifest.json"
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
if not manifest.get("configuration_shift"):
row["reason"] = "Dataset manifest has no declared configuration shift"
else:
row.update(status="completed", result=evaluator(config))
else:
row.update(status="completed", result=evaluator(config))
results.append(row)
Path(output).write_text(json.dumps({"experiments": results}, indent=2, allow_nan=False) + "\n")
return results