Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/evaluation/experiments.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 4.62 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/experiments.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/evaluation/experiments.py
-
curl -L -o experiments.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/experiments.py
4.62 kB
| """Explicit experiment provenance and observation-only stress transformations.""" | |
| from __future__ import annotations | |
| import json | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| class ExperimentConfig: | |
| name: str | |
| axis: str = "B" | |
| data: str | None = None | |
| checkpoint: str | None = None | |
| excluded_family: str | None = None | |
| sensor_failure_probability: float = 0.0 | |
| sensor_bias_std: float = 0.0 | |
| description: str = "" | |
| def validate(self): | |
| if self.axis not in ("B", "C", "D", "E"): | |
| raise ValueError("Experiment axis must be B, C, D or E") | |
| if not 0 <= self.sensor_failure_probability <= 1 or self.sensor_bias_std < 0: | |
| raise ValueError("Invalid sensor stress settings") | |
| if self.axis == "E" and not self.excluded_family: | |
| raise ValueError("Leave-family-out evaluation needs an excluded family") | |
| return self | |
| def default_experiments(): | |
| return [ | |
| ExperimentConfig("heldout_systems", "B", description="Main held-out-system test"), | |
| ExperimentConfig( | |
| "sensor_failures", | |
| "C", | |
| sensor_failure_probability=0.25, | |
| description="Observed history/current sensor dropout", | |
| ), | |
| ExperimentConfig( | |
| "sensor_bias", | |
| "C", | |
| sensor_bias_std=0.5, | |
| description="Observed state sensor bias in training-standardized units", | |
| ), | |
| ExperimentConfig( | |
| "configuration_shift", | |
| "D", | |
| description="Requires independently generated shifted physical parameters", | |
| ), | |
| *[ | |
| ExperimentConfig( | |
| f"leave_{family}_out", | |
| "E", | |
| excluded_family=family, | |
| description="Requires checkpoint trained with this family excluded", | |
| ) | |
| for family in ("thermal", "tank", "battery") | |
| ], | |
| ] | |
| def stress_observations(batch, config, seed=0): | |
| """Corrupt state observations, never targets or supplied future controls.""" | |
| config.validate() | |
| out = {k: v.clone() if torch.is_tensor(v) else v for k, v in batch.items()} | |
| rng = np.random.default_rng(seed) | |
| state = out["role_ids"] == 0 | |
| if config.sensor_failure_probability: | |
| dropped = ( | |
| torch.tensor(rng.random(state.shape) < config.sensor_failure_probability, device=state.device) | |
| & state | |
| ) | |
| out["history_mask"] = out["history_mask"] * (~dropped[:, None, :]) | |
| out["current_mask"] = out["current_mask"] * (~dropped) | |
| if config.sensor_bias_std: | |
| bias = ( | |
| torch.tensor( | |
| rng.normal(0, config.sensor_bias_std, state.shape), | |
| dtype=out["history"].dtype, | |
| device=state.device, | |
| ) | |
| * state | |
| ) | |
| out["history"] = out["history"] + bias[:, None, :] | |
| out["current"] = out["current"] + bias | |
| out["history"] *= out["history_mask"] | |
| out["current"] *= out["current_mask"] | |
| return out | |
| def run_experiments(configs, evaluator, output): | |
| """Evaluator receives validated config; absent prerequisites remain unrun.""" | |
| results = [] | |
| for config in configs: | |
| config.validate() | |
| row = {"config": asdict(config), "status": "unrun"} | |
| if not config.data or not config.checkpoint: | |
| row["reason"] = "Dataset and compatible trained checkpoint were not supplied" | |
| elif config.axis == "E": | |
| run_path = Path(config.checkpoint).parent / "run.json" | |
| run = json.loads(run_path.read_text()) if run_path.exists() else {} | |
| if config.excluded_family not in run.get("excluded_families", []): | |
| row["reason"] = "Checkpoint has no provenance proving the family was excluded from training" | |
| else: | |
| row.update(status="completed", result=evaluator(config)) | |
| elif config.axis == "D": | |
| manifest_path = Path(config.data) / "manifest.json" | |
| manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} | |
| if not manifest.get("configuration_shift"): | |
| row["reason"] = "Dataset manifest has no declared configuration shift" | |
| else: | |
| row.update(status="completed", result=evaluator(config)) | |
| else: | |
| row.update(status="completed", result=evaluator(config)) | |
| results.append(row) | |
| Path(output).write_text(json.dumps({"experiments": results}, indent=2, allow_nan=False) + "\n") | |
| return results | |