"""Explicit experiment provenance and observation-only stress transformations.""" from __future__ import annotations import json from dataclasses import asdict, dataclass from pathlib import Path import numpy as np import torch @dataclass class ExperimentConfig: name: str axis: str = "B" data: str | None = None checkpoint: str | None = None excluded_family: str | None = None sensor_failure_probability: float = 0.0 sensor_bias_std: float = 0.0 description: str = "" def validate(self): if self.axis not in ("B", "C", "D", "E"): raise ValueError("Experiment axis must be B, C, D or E") if not 0 <= self.sensor_failure_probability <= 1 or self.sensor_bias_std < 0: raise ValueError("Invalid sensor stress settings") if self.axis == "E" and not self.excluded_family: raise ValueError("Leave-family-out evaluation needs an excluded family") return self def default_experiments(): return [ ExperimentConfig("heldout_systems", "B", description="Main held-out-system test"), ExperimentConfig( "sensor_failures", "C", sensor_failure_probability=0.25, description="Observed history/current sensor dropout", ), ExperimentConfig( "sensor_bias", "C", sensor_bias_std=0.5, description="Observed state sensor bias in training-standardized units", ), ExperimentConfig( "configuration_shift", "D", description="Requires independently generated shifted physical parameters", ), *[ ExperimentConfig( f"leave_{family}_out", "E", excluded_family=family, description="Requires checkpoint trained with this family excluded", ) for family in ("thermal", "tank", "battery") ], ] def stress_observations(batch, config, seed=0): """Corrupt state observations, never targets or supplied future controls.""" config.validate() out = {k: v.clone() if torch.is_tensor(v) else v for k, v in batch.items()} rng = np.random.default_rng(seed) state = out["role_ids"] == 0 if config.sensor_failure_probability: dropped = ( torch.tensor(rng.random(state.shape) < config.sensor_failure_probability, device=state.device) & state ) out["history_mask"] = out["history_mask"] * (~dropped[:, None, :]) out["current_mask"] = out["current_mask"] * (~dropped) if config.sensor_bias_std: bias = ( torch.tensor( rng.normal(0, config.sensor_bias_std, state.shape), dtype=out["history"].dtype, device=state.device, ) * state ) out["history"] = out["history"] + bias[:, None, :] out["current"] = out["current"] + bias out["history"] *= out["history_mask"] out["current"] *= out["current_mask"] return out def run_experiments(configs, evaluator, output): """Evaluator receives validated config; absent prerequisites remain unrun.""" results = [] for config in configs: config.validate() row = {"config": asdict(config), "status": "unrun"} if not config.data or not config.checkpoint: row["reason"] = "Dataset and compatible trained checkpoint were not supplied" elif config.axis == "E": run_path = Path(config.checkpoint).parent / "run.json" run = json.loads(run_path.read_text()) if run_path.exists() else {} if config.excluded_family not in run.get("excluded_families", []): row["reason"] = "Checkpoint has no provenance proving the family was excluded from training" else: row.update(status="completed", result=evaluator(config)) elif config.axis == "D": manifest_path = Path(config.data) / "manifest.json" manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} if not manifest.get("configuration_shift"): row["reason"] = "Dataset manifest has no declared configuration shift" else: row.update(status="completed", result=evaluator(config)) else: row.update(status="completed", result=evaluator(config)) results.append(row) Path(output).write_text(json.dumps({"experiments": results}, indent=2, allow_nan=False) + "\n") return results