Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/evaluation/context.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/context.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/evaluation/context.py
-
curl -L -o context.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/context.py
10.6 kB
| """Compare history24/history96/no-history on exactly the same eligible anchors. | |
| All test anchors are sampled once from the history96-eligible population. Shorter | |
| histories use these same anchors with their own feature length and training-only | |
| normalization. Training windows naturally have different eligibility populations; | |
| this is disclosed and the training run budgets are recorded. Calibration is not | |
| applied to this ablation. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from tfm.data import Normalization, WindowDataset | |
| from tfm.training.train import load_checkpoint, load_manifest_episodes, resolve_device | |
| from .benchmark import collect_predictions, sampling_provenance, sha256, summarize | |
| def shared_anchor_datasets(episodes, normalizations, history_steps, *, samples=512, seed=2026): | |
| """Return separate feature views of one independently sampled anchor list.""" | |
| if set(normalizations) != set(history_steps) or not history_steps: | |
| raise ValueError("Each history view needs a matching normalization") | |
| longest = max(history_steps, key=history_steps.get) | |
| eligible = WindowDataset( | |
| episodes, | |
| normalizations[longest], | |
| history_steps=history_steps[longest], | |
| future_steps=8, | |
| samples=samples, | |
| seed=seed, | |
| ) | |
| datasets = {} | |
| for name, length in history_steps.items(): | |
| dataset = WindowDataset( | |
| episodes, normalizations[name], history_steps=length, future_steps=8, samples=samples, seed=seed | |
| ) | |
| dataset.indices = list(eligible.indices) | |
| datasets[name] = dataset | |
| return datasets | |
| def evaluate_context( | |
| data, | |
| h24_checkpoint, | |
| h96_checkpoint, | |
| no_history_checkpoint, | |
| output, | |
| *, | |
| samples=512, | |
| seed=2026, | |
| batch_size=64, | |
| device="cpu", | |
| ): | |
| manifest, splits = load_manifest_episodes(data) | |
| paths = { | |
| "history24": Path(h24_checkpoint), | |
| "history96": Path(h96_checkpoint), | |
| "no_history": Path(no_history_checkpoint), | |
| } | |
| selected_device = resolve_device(device) | |
| models, checkpoints, normalizations, training_runs = {}, {}, {}, {} | |
| train_ids = {str(episode.system_id) for episode in splits["train"]} | |
| test_ids = {str(episode.system_id) for episode in splits["test"]} | |
| for name, path in paths.items(): | |
| run_path, norm_path = path.parent / "run.json", path.parent / "normalization.json" | |
| if not run_path.exists(): | |
| raise ValueError(f"Context checkpoint training has not completed: {run_path}") | |
| model, checkpoint = load_checkpoint(path, selected_device) | |
| normalization = Normalization.load(norm_path) | |
| if checkpoint.get("normalization_sha256") != sha256(norm_path): | |
| raise ValueError("Context checkpoint normalization fingerprint mismatch") | |
| if ( | |
| set(checkpoint["training_system_ids"]) != train_ids | |
| or set(normalization.training_system_ids) != train_ids | |
| ): | |
| raise ValueError("Context training system provenance differs from the supplied data") | |
| if set(checkpoint["test_system_ids"]) != test_ids or test_ids & set( | |
| checkpoint["validation_selection_system_ids"] | |
| ): | |
| raise ValueError("Context test system provenance differs or overlaps model selection") | |
| models[name], checkpoints[name], normalizations[name] = model, checkpoint, normalization | |
| training_runs[name] = json.loads(run_path.read_text()) | |
| lengths = { | |
| name: int(checkpoint["model_config"].get("history_steps", 24)) | |
| for name, checkpoint in checkpoints.items() | |
| } | |
| if lengths != {"history24": 24, "history96": 96, "no_history": 24}: | |
| raise ValueError("Context ablation requires H24, H96, and H24 no-history checkpoints") | |
| if checkpoints["no_history"]["model_config"].get("use_history", True): | |
| raise ValueError("The no-history checkpoint does not prove history was disabled") | |
| if any( | |
| not checkpoints[name]["model_config"].get("use_history", True) for name in ("history24", "history96") | |
| ): | |
| raise ValueError("The history24/history96 checkpoints must enable history") | |
| reference = { | |
| key: value | |
| for key, value in checkpoints["history24"]["model_config"].items() | |
| if key not in ("history_steps", "use_history") | |
| } | |
| for checkpoint in checkpoints.values(): | |
| config = { | |
| key: value | |
| for key, value in checkpoint["model_config"].items() | |
| if key not in ("history_steps", "use_history") | |
| } | |
| if config != reference or checkpoint["model_kind"] != checkpoints["history24"]["model_kind"]: | |
| raise ValueError("Context models must share architecture apart from history configuration") | |
| for key in ( | |
| "training_system_ids", | |
| "validation_selection_system_ids", | |
| "calibration_system_ids", | |
| "test_system_ids", | |
| ): | |
| if set(checkpoint[key]) != set(checkpoints["history24"][key]): | |
| raise ValueError("Context checkpoints require identical system partitions") | |
| datasets = shared_anchor_datasets(splits["test"], normalizations, lengths, samples=samples, seed=seed) | |
| manifest_path = Path(data) | |
| if manifest_path.is_dir(): | |
| manifest_path = manifest_path / "manifest.json" | |
| result = { | |
| "version": "tfm-context-ablation-v1", | |
| "status": "completed", | |
| "seed": seed, | |
| "samples": samples, | |
| "device": str(selected_device), | |
| "generator_version": manifest["generator_version"], | |
| "manifest_sha256": sha256(manifest_path), | |
| "test_system_ids": sorted(test_ids), | |
| "shared_anchor_provenance": sampling_provenance(datasets["history96"], manifest_path)[ | |
| "window_sampling" | |
| ], | |
| "calibration_status": "not_applied_context_ablation", | |
| "limitation": "Same test anchors are eligible for H96 and evaluated by every model. H24/no-history and H96 training anchors have different eligibility populations; matching optimizer budgets does not make their training samples identical. This is a short synthetic context ablation, not a scaling or research claim.", | |
| "models": {}, | |
| } | |
| anchor_hashes = set() | |
| for name, model in models.items(): | |
| dataset, normalization, checkpoint, run = ( | |
| datasets[name], | |
| normalizations[name], | |
| checkpoints[name], | |
| training_runs[name], | |
| ) | |
| mean, log_std, arrays = collect_predictions(model, dataset, selected_device, batch_size) | |
| families = [str(dataset.episodes[episode].family_id) for episode, _ in dataset.indices] | |
| metrics = summarize(mean, log_std, arrays, families, normalization) | |
| provenance = sampling_provenance(dataset, manifest_path)["window_sampling"] | |
| anchor_hashes.add(provenance["anchor_sha256"]) | |
| arguments = run.get("arguments", {}) | |
| result["models"][name] = { | |
| "checkpoint_sha256": sha256(paths[name]), | |
| "normalization_sha256": sha256(paths[name].parent / "normalization.json"), | |
| "training_run_sha256": sha256(paths[name].parent / "run.json"), | |
| "model_kind": checkpoint["model_kind"], | |
| "model_config": checkpoint["model_config"], | |
| "parameter_count": sum(p.numel() for p in model.parameters()), | |
| "window_sampling": provenance, | |
| "training_seed": run.get("seed", checkpoint.get("seed")), | |
| "training_budget": { | |
| "requested_epochs": arguments.get("epochs"), | |
| "epochs_completed": run.get("epochs_completed"), | |
| "training_windows_per_epoch": arguments.get("samples"), | |
| "batch_size": arguments.get("batch_size"), | |
| "selected_checkpoint_epoch": checkpoint.get("epoch"), | |
| }, | |
| "metrics": metrics, | |
| "balanced_standardized_summary": [ | |
| { | |
| "family": family, | |
| "standardized_mae": float( | |
| np.mean([row["standardized_mae"] for row in metrics if row["family"] == family]) | |
| ), | |
| "standardized_rmse": float( | |
| np.mean([row["standardized_rmse"] for row in metrics if row["family"] == family]) | |
| ), | |
| "standardized_gaussian_nll": float( | |
| np.mean( | |
| [row["standardized_gaussian_nll"] for row in metrics if row["family"] == family] | |
| ) | |
| ), | |
| "observed_query_count": sum(row["count"] for row in metrics if row["family"] == family), | |
| "weighting": "equal field/horizon row weight", | |
| } | |
| for family in sorted(set(families)) | |
| ], | |
| } | |
| if len(anchor_hashes) != 1: | |
| raise RuntimeError("Context ablation anchor alignment failed") | |
| budgets = [row["training_budget"] for row in result["models"].values()] | |
| result["training_optimizer_budgets_match"] = all( | |
| {k: v for k, v in budget.items() if k != "selected_checkpoint_epoch"} | |
| == {k: v for k, v in budgets[0].items() if k != "selected_checkpoint_epoch"} | |
| for budget in budgets | |
| ) | |
| output = Path(output) | |
| output.mkdir(parents=True, exist_ok=True) | |
| (output / "context.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") | |
| return result | |
| def main(argv=None): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--data", required=True) | |
| parser.add_argument("--h24-checkpoint", required=True) | |
| parser.add_argument("--h96-checkpoint", required=True) | |
| parser.add_argument("--no-history-checkpoint", required=True) | |
| parser.add_argument("--output", required=True) | |
| parser.add_argument("--samples", type=int, default=512) | |
| parser.add_argument("--seed", type=int, default=2026) | |
| parser.add_argument("--batch-size", type=int, default=64) | |
| parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="cpu") | |
| args = parser.parse_args(argv) | |
| if min(args.samples, args.batch_size) < 1: | |
| raise ValueError("Context sample and batch counts must be positive") | |
| evaluate_context( | |
| args.data, | |
| args.h24_checkpoint, | |
| args.h96_checkpoint, | |
| args.no_history_checkpoint, | |
| args.output, | |
| samples=args.samples, | |
| seed=args.seed, | |
| batch_size=args.batch_size, | |
| device=args.device, | |
| ) | |
| if __name__ == "__main__": | |
| main() | |