maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
10.6 kB
"""Compare history24/history96/no-history on exactly the same eligible anchors.
All test anchors are sampled once from the history96-eligible population. Shorter
histories use these same anchors with their own feature length and training-only
normalization. Training windows naturally have different eligibility populations;
this is disclosed and the training run budgets are recorded. Calibration is not
applied to this ablation.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import numpy as np
from tfm.data import Normalization, WindowDataset
from tfm.training.train import load_checkpoint, load_manifest_episodes, resolve_device
from .benchmark import collect_predictions, sampling_provenance, sha256, summarize
def shared_anchor_datasets(episodes, normalizations, history_steps, *, samples=512, seed=2026):
"""Return separate feature views of one independently sampled anchor list."""
if set(normalizations) != set(history_steps) or not history_steps:
raise ValueError("Each history view needs a matching normalization")
longest = max(history_steps, key=history_steps.get)
eligible = WindowDataset(
episodes,
normalizations[longest],
history_steps=history_steps[longest],
future_steps=8,
samples=samples,
seed=seed,
)
datasets = {}
for name, length in history_steps.items():
dataset = WindowDataset(
episodes, normalizations[name], history_steps=length, future_steps=8, samples=samples, seed=seed
)
dataset.indices = list(eligible.indices)
datasets[name] = dataset
return datasets
def evaluate_context(
data,
h24_checkpoint,
h96_checkpoint,
no_history_checkpoint,
output,
*,
samples=512,
seed=2026,
batch_size=64,
device="cpu",
):
manifest, splits = load_manifest_episodes(data)
paths = {
"history24": Path(h24_checkpoint),
"history96": Path(h96_checkpoint),
"no_history": Path(no_history_checkpoint),
}
selected_device = resolve_device(device)
models, checkpoints, normalizations, training_runs = {}, {}, {}, {}
train_ids = {str(episode.system_id) for episode in splits["train"]}
test_ids = {str(episode.system_id) for episode in splits["test"]}
for name, path in paths.items():
run_path, norm_path = path.parent / "run.json", path.parent / "normalization.json"
if not run_path.exists():
raise ValueError(f"Context checkpoint training has not completed: {run_path}")
model, checkpoint = load_checkpoint(path, selected_device)
normalization = Normalization.load(norm_path)
if checkpoint.get("normalization_sha256") != sha256(norm_path):
raise ValueError("Context checkpoint normalization fingerprint mismatch")
if (
set(checkpoint["training_system_ids"]) != train_ids
or set(normalization.training_system_ids) != train_ids
):
raise ValueError("Context training system provenance differs from the supplied data")
if set(checkpoint["test_system_ids"]) != test_ids or test_ids & set(
checkpoint["validation_selection_system_ids"]
):
raise ValueError("Context test system provenance differs or overlaps model selection")
models[name], checkpoints[name], normalizations[name] = model, checkpoint, normalization
training_runs[name] = json.loads(run_path.read_text())
lengths = {
name: int(checkpoint["model_config"].get("history_steps", 24))
for name, checkpoint in checkpoints.items()
}
if lengths != {"history24": 24, "history96": 96, "no_history": 24}:
raise ValueError("Context ablation requires H24, H96, and H24 no-history checkpoints")
if checkpoints["no_history"]["model_config"].get("use_history", True):
raise ValueError("The no-history checkpoint does not prove history was disabled")
if any(
not checkpoints[name]["model_config"].get("use_history", True) for name in ("history24", "history96")
):
raise ValueError("The history24/history96 checkpoints must enable history")
reference = {
key: value
for key, value in checkpoints["history24"]["model_config"].items()
if key not in ("history_steps", "use_history")
}
for checkpoint in checkpoints.values():
config = {
key: value
for key, value in checkpoint["model_config"].items()
if key not in ("history_steps", "use_history")
}
if config != reference or checkpoint["model_kind"] != checkpoints["history24"]["model_kind"]:
raise ValueError("Context models must share architecture apart from history configuration")
for key in (
"training_system_ids",
"validation_selection_system_ids",
"calibration_system_ids",
"test_system_ids",
):
if set(checkpoint[key]) != set(checkpoints["history24"][key]):
raise ValueError("Context checkpoints require identical system partitions")
datasets = shared_anchor_datasets(splits["test"], normalizations, lengths, samples=samples, seed=seed)
manifest_path = Path(data)
if manifest_path.is_dir():
manifest_path = manifest_path / "manifest.json"
result = {
"version": "tfm-context-ablation-v1",
"status": "completed",
"seed": seed,
"samples": samples,
"device": str(selected_device),
"generator_version": manifest["generator_version"],
"manifest_sha256": sha256(manifest_path),
"test_system_ids": sorted(test_ids),
"shared_anchor_provenance": sampling_provenance(datasets["history96"], manifest_path)[
"window_sampling"
],
"calibration_status": "not_applied_context_ablation",
"limitation": "Same test anchors are eligible for H96 and evaluated by every model. H24/no-history and H96 training anchors have different eligibility populations; matching optimizer budgets does not make their training samples identical. This is a short synthetic context ablation, not a scaling or research claim.",
"models": {},
}
anchor_hashes = set()
for name, model in models.items():
dataset, normalization, checkpoint, run = (
datasets[name],
normalizations[name],
checkpoints[name],
training_runs[name],
)
mean, log_std, arrays = collect_predictions(model, dataset, selected_device, batch_size)
families = [str(dataset.episodes[episode].family_id) for episode, _ in dataset.indices]
metrics = summarize(mean, log_std, arrays, families, normalization)
provenance = sampling_provenance(dataset, manifest_path)["window_sampling"]
anchor_hashes.add(provenance["anchor_sha256"])
arguments = run.get("arguments", {})
result["models"][name] = {
"checkpoint_sha256": sha256(paths[name]),
"normalization_sha256": sha256(paths[name].parent / "normalization.json"),
"training_run_sha256": sha256(paths[name].parent / "run.json"),
"model_kind": checkpoint["model_kind"],
"model_config": checkpoint["model_config"],
"parameter_count": sum(p.numel() for p in model.parameters()),
"window_sampling": provenance,
"training_seed": run.get("seed", checkpoint.get("seed")),
"training_budget": {
"requested_epochs": arguments.get("epochs"),
"epochs_completed": run.get("epochs_completed"),
"training_windows_per_epoch": arguments.get("samples"),
"batch_size": arguments.get("batch_size"),
"selected_checkpoint_epoch": checkpoint.get("epoch"),
},
"metrics": metrics,
"balanced_standardized_summary": [
{
"family": family,
"standardized_mae": float(
np.mean([row["standardized_mae"] for row in metrics if row["family"] == family])
),
"standardized_rmse": float(
np.mean([row["standardized_rmse"] for row in metrics if row["family"] == family])
),
"standardized_gaussian_nll": float(
np.mean(
[row["standardized_gaussian_nll"] for row in metrics if row["family"] == family]
)
),
"observed_query_count": sum(row["count"] for row in metrics if row["family"] == family),
"weighting": "equal field/horizon row weight",
}
for family in sorted(set(families))
],
}
if len(anchor_hashes) != 1:
raise RuntimeError("Context ablation anchor alignment failed")
budgets = [row["training_budget"] for row in result["models"].values()]
result["training_optimizer_budgets_match"] = all(
{k: v for k, v in budget.items() if k != "selected_checkpoint_epoch"}
== {k: v for k, v in budgets[0].items() if k != "selected_checkpoint_epoch"}
for budget in budgets
)
output = Path(output)
output.mkdir(parents=True, exist_ok=True)
(output / "context.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
return result
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data", required=True)
parser.add_argument("--h24-checkpoint", required=True)
parser.add_argument("--h96-checkpoint", required=True)
parser.add_argument("--no-history-checkpoint", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--samples", type=int, default=512)
parser.add_argument("--seed", type=int, default=2026)
parser.add_argument("--batch-size", type=int, default=64)
parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="cpu")
args = parser.parse_args(argv)
if min(args.samples, args.batch_size) < 1:
raise ValueError("Context sample and batch counts must be positive")
evaluate_context(
args.data,
args.h24_checkpoint,
args.h96_checkpoint,
args.no_history_checkpoint,
args.output,
samples=args.samples,
seed=args.seed,
batch_size=args.batch_size,
device=args.device,
)
if __name__ == "__main__":
main()