Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/evaluation/comparison.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/comparison.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/evaluation/comparison.py
-
curl -L -o comparison.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/comparison.py
10.8 kB
| """Verify and summarize aligned benchmark reports without pooling physical units. | |
| Python API: build_comparison(data_path, runs, output), where runs maps each name | |
| to {"report": benchmark-directory, "checkpoints": [checkpoint-path, ...]}. | |
| CLI: python -m tfm.evaluation.comparison --data DATA --config RUNS.json --output comparison.json | |
| Missing reports are explicitly unrun. Re-running refreshes report fingerprints | |
| after isolated latency measurements have been added to existing reports. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from tfm.data import FIELD_INDEX, Normalization, WindowDataset | |
| from tfm.training.train import load_checkpoint, load_manifest_episodes | |
| from .benchmark import sampling_provenance, sha256 | |
| from .calibration import validate_calibration | |
| def _summarize_model(model, normalization): | |
| if model["status"] != "completed": | |
| return {"status": model["status"], "reason": model.get("reason")} | |
| forecasts = [] | |
| for family in sorted({row["family"] for row in model["metrics"]}): | |
| rows = [row for row in model["metrics"] if row["family"] == family] | |
| calibrated = [row for row in rows if row["interval_kind"] == "heldout_calibrated"] | |
| calibrated_count = sum(row["count"] for row in calibrated) | |
| count = sum(row["count"] for row in rows) | |
| forecasts.append( | |
| { | |
| "family": family, | |
| "field_horizon_balanced_standardized_mae": float( | |
| np.mean([row["standardized_mae"] for row in rows]) | |
| ), | |
| "coverage_weighted_by_count": sum(row["coverage"] * row["count"] for row in calibrated) | |
| / calibrated_count | |
| if calibrated_count | |
| else None, | |
| "calibrated_query_count": calibrated_count, | |
| "total_observed_query_count": count, | |
| "calibrated_query_fraction": calibrated_count / count if count else 0, | |
| "field_horizon_balanced_standardized_interval_width": float( | |
| np.mean( | |
| [ | |
| row["mean_interval_width"] / normalization.scale[row["field_id"]] | |
| for row in calibrated | |
| ] | |
| ) | |
| ) | |
| if calibrated | |
| else None, | |
| "uncalibrated_fields": sorted( | |
| {row["field"] for row in rows if row["interval_kind"] != "heldout_calibrated"} | |
| ), | |
| } | |
| ) | |
| interventions = [] | |
| for row in model.get("counterfactual", {}).get("summary_by_family", []): | |
| interventions.append( | |
| { | |
| "family": row["family"], | |
| "sample_count": row["sample_count"], | |
| "field_horizon_balanced_standardized_action_effect_mae": float( | |
| np.mean( | |
| [ | |
| effect["action_effect_mae"] / normalization.scale[FIELD_INDEX[effect["field"]]] | |
| for effect in row["effects"] | |
| ] | |
| ) | |
| ), | |
| "ranking_accuracy": row["ranking_accuracy"], | |
| "mean_regret_family_objective_units": row["mean_regret"], | |
| } | |
| ) | |
| return { | |
| "forecast": forecasts, | |
| "intervention": interventions, | |
| "calibration_status": model.get("calibration_status"), | |
| } | |
| def build_comparison(data_path, runs, output): | |
| data_path = Path(data_path) | |
| manifest_path = data_path / "manifest.json" if data_path.is_dir() else data_path | |
| manifest, splits = load_manifest_episodes(data_path) | |
| manifest_ids = {key: {str(ep.system_id) for ep in episodes} for key, episodes in splits.items()} | |
| result = { | |
| "version": "tfm-comparison-v1", | |
| "status": "smoke_only", | |
| "generator_version": manifest["generator_version"], | |
| "manifest_sha256": sha256(manifest_path), | |
| "partition_episode_counts": {key: len(value) for key, value in splits.items()}, | |
| "interpretation": "Short synthetic runs, no research or scaling claim. Prediction and action-effect MAE equally average field/horizon rows within family. Coverage is weighted by calibrated observed query counts and omits unsupported fields, which are listed. Widths divide by each field's training standard deviation; physical widths remain in source reports. Regret has family-specific objective units and must not be pooled. Latency retains the source measurement scope and contention limitations.", | |
| "runs": {}, | |
| } | |
| reference_sampling, baseline_reference = None, {} | |
| for name, specification in runs.items(): | |
| report_path = Path(specification["report"]) | |
| if report_path.is_dir() or not report_path.suffix: | |
| report_path = report_path / "benchmark.json" | |
| if not report_path.exists(): | |
| result["runs"][name] = {"status": "unrun", "reason": "Benchmark report not yet available"} | |
| continue | |
| report = json.loads(report_path.read_text()) | |
| paths = [Path(path) for path in specification["checkpoints"]] | |
| checkpoints = [load_checkpoint(path)[1] for path in paths] | |
| checkpoint = checkpoints[0] | |
| norm_path = paths[0].parent / "normalization.json" | |
| normalization = Normalization.load(norm_path) | |
| if report["checkpoint_sha256"] != [sha256(path) for path in paths]: | |
| raise ValueError("Comparison report checkpoint fingerprint mismatch") | |
| if ( | |
| report["normalization_sha256"] != sha256(norm_path) | |
| or report["manifest_sha256"] != result["manifest_sha256"] | |
| ): | |
| raise ValueError("Comparison normalization or dataset fingerprint mismatch") | |
| keys = ( | |
| "training_system_ids", | |
| "validation_selection_system_ids", | |
| "calibration_system_ids", | |
| "test_system_ids", | |
| ) | |
| groups = {key: set(checkpoint[key]) for key in keys} | |
| for i, key in enumerate(keys): | |
| if any(groups[key] & groups[other] for other in keys[:i]): | |
| raise ValueError("Comparison checkpoint partition leakage") | |
| if any(set(member[key]) != groups[key] for member in checkpoints): | |
| raise ValueError("Comparison ensemble member partitions differ") | |
| if ( | |
| groups["training_system_ids"] != manifest_ids["train"] | |
| or groups["test_system_ids"] != manifest_ids["test"] | |
| ): | |
| raise ValueError("Comparison checkpoints differ from manifest training or test partitions") | |
| if ( | |
| groups["validation_selection_system_ids"] | groups["calibration_system_ids"] | |
| != manifest_ids["validation"] | |
| ): | |
| raise ValueError("Comparison validation partition provenance differs") | |
| if set(normalization.training_system_ids) != groups["training_system_ids"]: | |
| raise ValueError("Comparison normalization training provenance differs") | |
| if set(report["test_system_ids"]) != groups["test_system_ids"]: | |
| raise ValueError("Comparison report test provenance differs") | |
| cal_path = report_path.parent / "calibration.json" | |
| calibration = json.loads(cal_path.read_text()) | |
| validate_calibration( | |
| calibration, | |
| checkpoint_sha256=report["checkpoint_sha256"], | |
| normalization_version=1, | |
| normalization_sha256=report["normalization_sha256"], | |
| forbidden_system_ids=groups["training_system_ids"] | |
| | groups["validation_selection_system_ids"] | |
| | groups["test_system_ids"], | |
| ) | |
| if set(calibration["calibration_system_ids"]) != groups["calibration_system_ids"]: | |
| raise ValueError("Comparison calibration systems differ from reserved partition") | |
| dataset = WindowDataset( | |
| splits["test"], | |
| normalization, | |
| history_steps=checkpoint["model_config"]["history_steps"], | |
| future_steps=8, | |
| samples=report["samples"], | |
| seed=report["seed"], | |
| ) | |
| expected = sampling_provenance(dataset, manifest_path)["window_sampling"] | |
| if ( | |
| report["window_sampling"] != expected | |
| or reference_sampling is not None | |
| and expected != reference_sampling | |
| ): | |
| raise ValueError("Comparison sampled windows differ") | |
| reference_sampling = expected | |
| models = { | |
| model_name: _summarize_model(model, normalization) | |
| for model_name, model in report["models"].items() | |
| } | |
| for baseline in ("persistence", "history_linear_system_id"): | |
| if baseline in baseline_reference and models[baseline] != baseline_reference[baseline]: | |
| raise ValueError("Comparison aligned baseline results differ") | |
| baseline_reference[baseline] = models[baseline] | |
| generalization = json.loads((report_path.parent / "generalization.json").read_text()) | |
| result["runs"][name] = { | |
| "status": "completed", | |
| "provenance_checks": "passed", | |
| "source_report": str(report_path), | |
| "source_report_sha256": sha256(report_path), | |
| "calibration_sha256": sha256(cal_path), | |
| "checkpoint_sha256": report["checkpoint_sha256"], | |
| "normalization_sha256": report["normalization_sha256"], | |
| "split_system_counts": {key: len(ids) for key, ids in groups.items()}, | |
| "parameter_count": report["parameter_count"], | |
| "inference": report["inference"], | |
| "window_sampling": expected, | |
| "models": models, | |
| "generalization": [ | |
| { | |
| "name": exp["config"]["name"], | |
| "axis": exp["config"]["axis"], | |
| "status": exp["status"], | |
| "reason": exp.get("reason"), | |
| "evaluated_target_fraction": exp.get("result", {}).get("evaluated_target_fraction"), | |
| } | |
| for exp in generalization["experiments"] | |
| ], | |
| } | |
| Path(output).write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") | |
| return result | |
| def main(argv=None): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--data", required=True) | |
| parser.add_argument( | |
| "--config", | |
| required=True, | |
| help="JSON mapping of names to report directories and checkpoint path lists", | |
| ) | |
| parser.add_argument("--output", required=True) | |
| args = parser.parse_args(argv) | |
| build_comparison(args.data, json.loads(Path(args.config).read_text()), args.output) | |
| if __name__ == "__main__": | |
| main() | |