"""Compare history24/history96/no-history on exactly the same eligible anchors. All test anchors are sampled once from the history96-eligible population. Shorter histories use these same anchors with their own feature length and training-only normalization. Training windows naturally have different eligibility populations; this is disclosed and the training run budgets are recorded. Calibration is not applied to this ablation. """ from __future__ import annotations import argparse import json from pathlib import Path import numpy as np from tfm.data import Normalization, WindowDataset from tfm.training.train import load_checkpoint, load_manifest_episodes, resolve_device from .benchmark import collect_predictions, sampling_provenance, sha256, summarize def shared_anchor_datasets(episodes, normalizations, history_steps, *, samples=512, seed=2026): """Return separate feature views of one independently sampled anchor list.""" if set(normalizations) != set(history_steps) or not history_steps: raise ValueError("Each history view needs a matching normalization") longest = max(history_steps, key=history_steps.get) eligible = WindowDataset( episodes, normalizations[longest], history_steps=history_steps[longest], future_steps=8, samples=samples, seed=seed, ) datasets = {} for name, length in history_steps.items(): dataset = WindowDataset( episodes, normalizations[name], history_steps=length, future_steps=8, samples=samples, seed=seed ) dataset.indices = list(eligible.indices) datasets[name] = dataset return datasets def evaluate_context( data, h24_checkpoint, h96_checkpoint, no_history_checkpoint, output, *, samples=512, seed=2026, batch_size=64, device="cpu", ): manifest, splits = load_manifest_episodes(data) paths = { "history24": Path(h24_checkpoint), "history96": Path(h96_checkpoint), "no_history": Path(no_history_checkpoint), } selected_device = resolve_device(device) models, checkpoints, normalizations, training_runs = {}, {}, {}, {} train_ids = {str(episode.system_id) for episode in splits["train"]} test_ids = {str(episode.system_id) for episode in splits["test"]} for name, path in paths.items(): run_path, norm_path = path.parent / "run.json", path.parent / "normalization.json" if not run_path.exists(): raise ValueError(f"Context checkpoint training has not completed: {run_path}") model, checkpoint = load_checkpoint(path, selected_device) normalization = Normalization.load(norm_path) if checkpoint.get("normalization_sha256") != sha256(norm_path): raise ValueError("Context checkpoint normalization fingerprint mismatch") if ( set(checkpoint["training_system_ids"]) != train_ids or set(normalization.training_system_ids) != train_ids ): raise ValueError("Context training system provenance differs from the supplied data") if set(checkpoint["test_system_ids"]) != test_ids or test_ids & set( checkpoint["validation_selection_system_ids"] ): raise ValueError("Context test system provenance differs or overlaps model selection") models[name], checkpoints[name], normalizations[name] = model, checkpoint, normalization training_runs[name] = json.loads(run_path.read_text()) lengths = { name: int(checkpoint["model_config"].get("history_steps", 24)) for name, checkpoint in checkpoints.items() } if lengths != {"history24": 24, "history96": 96, "no_history": 24}: raise ValueError("Context ablation requires H24, H96, and H24 no-history checkpoints") if checkpoints["no_history"]["model_config"].get("use_history", True): raise ValueError("The no-history checkpoint does not prove history was disabled") if any( not checkpoints[name]["model_config"].get("use_history", True) for name in ("history24", "history96") ): raise ValueError("The history24/history96 checkpoints must enable history") reference = { key: value for key, value in checkpoints["history24"]["model_config"].items() if key not in ("history_steps", "use_history") } for checkpoint in checkpoints.values(): config = { key: value for key, value in checkpoint["model_config"].items() if key not in ("history_steps", "use_history") } if config != reference or checkpoint["model_kind"] != checkpoints["history24"]["model_kind"]: raise ValueError("Context models must share architecture apart from history configuration") for key in ( "training_system_ids", "validation_selection_system_ids", "calibration_system_ids", "test_system_ids", ): if set(checkpoint[key]) != set(checkpoints["history24"][key]): raise ValueError("Context checkpoints require identical system partitions") datasets = shared_anchor_datasets(splits["test"], normalizations, lengths, samples=samples, seed=seed) manifest_path = Path(data) if manifest_path.is_dir(): manifest_path = manifest_path / "manifest.json" result = { "version": "tfm-context-ablation-v1", "status": "completed", "seed": seed, "samples": samples, "device": str(selected_device), "generator_version": manifest["generator_version"], "manifest_sha256": sha256(manifest_path), "test_system_ids": sorted(test_ids), "shared_anchor_provenance": sampling_provenance(datasets["history96"], manifest_path)[ "window_sampling" ], "calibration_status": "not_applied_context_ablation", "limitation": "Same test anchors are eligible for H96 and evaluated by every model. H24/no-history and H96 training anchors have different eligibility populations; matching optimizer budgets does not make their training samples identical. This is a short synthetic context ablation, not a scaling or research claim.", "models": {}, } anchor_hashes = set() for name, model in models.items(): dataset, normalization, checkpoint, run = ( datasets[name], normalizations[name], checkpoints[name], training_runs[name], ) mean, log_std, arrays = collect_predictions(model, dataset, selected_device, batch_size) families = [str(dataset.episodes[episode].family_id) for episode, _ in dataset.indices] metrics = summarize(mean, log_std, arrays, families, normalization) provenance = sampling_provenance(dataset, manifest_path)["window_sampling"] anchor_hashes.add(provenance["anchor_sha256"]) arguments = run.get("arguments", {}) result["models"][name] = { "checkpoint_sha256": sha256(paths[name]), "normalization_sha256": sha256(paths[name].parent / "normalization.json"), "training_run_sha256": sha256(paths[name].parent / "run.json"), "model_kind": checkpoint["model_kind"], "model_config": checkpoint["model_config"], "parameter_count": sum(p.numel() for p in model.parameters()), "window_sampling": provenance, "training_seed": run.get("seed", checkpoint.get("seed")), "training_budget": { "requested_epochs": arguments.get("epochs"), "epochs_completed": run.get("epochs_completed"), "training_windows_per_epoch": arguments.get("samples"), "batch_size": arguments.get("batch_size"), "selected_checkpoint_epoch": checkpoint.get("epoch"), }, "metrics": metrics, "balanced_standardized_summary": [ { "family": family, "standardized_mae": float( np.mean([row["standardized_mae"] for row in metrics if row["family"] == family]) ), "standardized_rmse": float( np.mean([row["standardized_rmse"] for row in metrics if row["family"] == family]) ), "standardized_gaussian_nll": float( np.mean( [row["standardized_gaussian_nll"] for row in metrics if row["family"] == family] ) ), "observed_query_count": sum(row["count"] for row in metrics if row["family"] == family), "weighting": "equal field/horizon row weight", } for family in sorted(set(families)) ], } if len(anchor_hashes) != 1: raise RuntimeError("Context ablation anchor alignment failed") budgets = [row["training_budget"] for row in result["models"].values()] result["training_optimizer_budgets_match"] = all( {k: v for k, v in budget.items() if k != "selected_checkpoint_epoch"} == {k: v for k, v in budgets[0].items() if k != "selected_checkpoint_epoch"} for budget in budgets ) output = Path(output) output.mkdir(parents=True, exist_ok=True) (output / "context.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") return result def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data", required=True) parser.add_argument("--h24-checkpoint", required=True) parser.add_argument("--h96-checkpoint", required=True) parser.add_argument("--no-history-checkpoint", required=True) parser.add_argument("--output", required=True) parser.add_argument("--samples", type=int, default=512) parser.add_argument("--seed", type=int, default=2026) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="cpu") args = parser.parse_args(argv) if min(args.samples, args.batch_size) < 1: raise ValueError("Context sample and batch counts must be positive") evaluate_context( args.data, args.h24_checkpoint, args.h96_checkpoint, args.no_history_checkpoint, args.output, samples=args.samples, seed=args.seed, batch_size=args.batch_size, device=args.device, ) if __name__ == "__main__": main()