File size: 10,633 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
"""Compare history24/history96/no-history on exactly the same eligible anchors.

All test anchors are sampled once from the history96-eligible population. Shorter
histories use these same anchors with their own feature length and training-only
normalization. Training windows naturally have different eligibility populations;
this is disclosed and the training run budgets are recorded. Calibration is not
applied to this ablation.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import numpy as np

from tfm.data import Normalization, WindowDataset
from tfm.training.train import load_checkpoint, load_manifest_episodes, resolve_device

from .benchmark import collect_predictions, sampling_provenance, sha256, summarize


def shared_anchor_datasets(episodes, normalizations, history_steps, *, samples=512, seed=2026):
    """Return separate feature views of one independently sampled anchor list."""
    if set(normalizations) != set(history_steps) or not history_steps:
        raise ValueError("Each history view needs a matching normalization")
    longest = max(history_steps, key=history_steps.get)
    eligible = WindowDataset(
        episodes,
        normalizations[longest],
        history_steps=history_steps[longest],
        future_steps=8,
        samples=samples,
        seed=seed,
    )
    datasets = {}
    for name, length in history_steps.items():
        dataset = WindowDataset(
            episodes, normalizations[name], history_steps=length, future_steps=8, samples=samples, seed=seed
        )
        dataset.indices = list(eligible.indices)
        datasets[name] = dataset
    return datasets


def evaluate_context(
    data,
    h24_checkpoint,
    h96_checkpoint,
    no_history_checkpoint,
    output,
    *,
    samples=512,
    seed=2026,
    batch_size=64,
    device="cpu",
):
    manifest, splits = load_manifest_episodes(data)
    paths = {
        "history24": Path(h24_checkpoint),
        "history96": Path(h96_checkpoint),
        "no_history": Path(no_history_checkpoint),
    }
    selected_device = resolve_device(device)
    models, checkpoints, normalizations, training_runs = {}, {}, {}, {}
    train_ids = {str(episode.system_id) for episode in splits["train"]}
    test_ids = {str(episode.system_id) for episode in splits["test"]}
    for name, path in paths.items():
        run_path, norm_path = path.parent / "run.json", path.parent / "normalization.json"
        if not run_path.exists():
            raise ValueError(f"Context checkpoint training has not completed: {run_path}")
        model, checkpoint = load_checkpoint(path, selected_device)
        normalization = Normalization.load(norm_path)
        if checkpoint.get("normalization_sha256") != sha256(norm_path):
            raise ValueError("Context checkpoint normalization fingerprint mismatch")
        if (
            set(checkpoint["training_system_ids"]) != train_ids
            or set(normalization.training_system_ids) != train_ids
        ):
            raise ValueError("Context training system provenance differs from the supplied data")
        if set(checkpoint["test_system_ids"]) != test_ids or test_ids & set(
            checkpoint["validation_selection_system_ids"]
        ):
            raise ValueError("Context test system provenance differs or overlaps model selection")
        models[name], checkpoints[name], normalizations[name] = model, checkpoint, normalization
        training_runs[name] = json.loads(run_path.read_text())
    lengths = {
        name: int(checkpoint["model_config"].get("history_steps", 24))
        for name, checkpoint in checkpoints.items()
    }
    if lengths != {"history24": 24, "history96": 96, "no_history": 24}:
        raise ValueError("Context ablation requires H24, H96, and H24 no-history checkpoints")
    if checkpoints["no_history"]["model_config"].get("use_history", True):
        raise ValueError("The no-history checkpoint does not prove history was disabled")
    if any(
        not checkpoints[name]["model_config"].get("use_history", True) for name in ("history24", "history96")
    ):
        raise ValueError("The history24/history96 checkpoints must enable history")
    reference = {
        key: value
        for key, value in checkpoints["history24"]["model_config"].items()
        if key not in ("history_steps", "use_history")
    }
    for checkpoint in checkpoints.values():
        config = {
            key: value
            for key, value in checkpoint["model_config"].items()
            if key not in ("history_steps", "use_history")
        }
        if config != reference or checkpoint["model_kind"] != checkpoints["history24"]["model_kind"]:
            raise ValueError("Context models must share architecture apart from history configuration")
        for key in (
            "training_system_ids",
            "validation_selection_system_ids",
            "calibration_system_ids",
            "test_system_ids",
        ):
            if set(checkpoint[key]) != set(checkpoints["history24"][key]):
                raise ValueError("Context checkpoints require identical system partitions")
    datasets = shared_anchor_datasets(splits["test"], normalizations, lengths, samples=samples, seed=seed)
    manifest_path = Path(data)
    if manifest_path.is_dir():
        manifest_path = manifest_path / "manifest.json"
    result = {
        "version": "tfm-context-ablation-v1",
        "status": "completed",
        "seed": seed,
        "samples": samples,
        "device": str(selected_device),
        "generator_version": manifest["generator_version"],
        "manifest_sha256": sha256(manifest_path),
        "test_system_ids": sorted(test_ids),
        "shared_anchor_provenance": sampling_provenance(datasets["history96"], manifest_path)[
            "window_sampling"
        ],
        "calibration_status": "not_applied_context_ablation",
        "limitation": "Same test anchors are eligible for H96 and evaluated by every model. H24/no-history and H96 training anchors have different eligibility populations; matching optimizer budgets does not make their training samples identical. This is a short synthetic context ablation, not a scaling or research claim.",
        "models": {},
    }
    anchor_hashes = set()
    for name, model in models.items():
        dataset, normalization, checkpoint, run = (
            datasets[name],
            normalizations[name],
            checkpoints[name],
            training_runs[name],
        )
        mean, log_std, arrays = collect_predictions(model, dataset, selected_device, batch_size)
        families = [str(dataset.episodes[episode].family_id) for episode, _ in dataset.indices]
        metrics = summarize(mean, log_std, arrays, families, normalization)
        provenance = sampling_provenance(dataset, manifest_path)["window_sampling"]
        anchor_hashes.add(provenance["anchor_sha256"])
        arguments = run.get("arguments", {})
        result["models"][name] = {
            "checkpoint_sha256": sha256(paths[name]),
            "normalization_sha256": sha256(paths[name].parent / "normalization.json"),
            "training_run_sha256": sha256(paths[name].parent / "run.json"),
            "model_kind": checkpoint["model_kind"],
            "model_config": checkpoint["model_config"],
            "parameter_count": sum(p.numel() for p in model.parameters()),
            "window_sampling": provenance,
            "training_seed": run.get("seed", checkpoint.get("seed")),
            "training_budget": {
                "requested_epochs": arguments.get("epochs"),
                "epochs_completed": run.get("epochs_completed"),
                "training_windows_per_epoch": arguments.get("samples"),
                "batch_size": arguments.get("batch_size"),
                "selected_checkpoint_epoch": checkpoint.get("epoch"),
            },
            "metrics": metrics,
            "balanced_standardized_summary": [
                {
                    "family": family,
                    "standardized_mae": float(
                        np.mean([row["standardized_mae"] for row in metrics if row["family"] == family])
                    ),
                    "standardized_rmse": float(
                        np.mean([row["standardized_rmse"] for row in metrics if row["family"] == family])
                    ),
                    "standardized_gaussian_nll": float(
                        np.mean(
                            [row["standardized_gaussian_nll"] for row in metrics if row["family"] == family]
                        )
                    ),
                    "observed_query_count": sum(row["count"] for row in metrics if row["family"] == family),
                    "weighting": "equal field/horizon row weight",
                }
                for family in sorted(set(families))
            ],
        }
    if len(anchor_hashes) != 1:
        raise RuntimeError("Context ablation anchor alignment failed")
    budgets = [row["training_budget"] for row in result["models"].values()]
    result["training_optimizer_budgets_match"] = all(
        {k: v for k, v in budget.items() if k != "selected_checkpoint_epoch"}
        == {k: v for k, v in budgets[0].items() if k != "selected_checkpoint_epoch"}
        for budget in budgets
    )
    output = Path(output)
    output.mkdir(parents=True, exist_ok=True)
    (output / "context.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
    return result


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--data", required=True)
    parser.add_argument("--h24-checkpoint", required=True)
    parser.add_argument("--h96-checkpoint", required=True)
    parser.add_argument("--no-history-checkpoint", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--samples", type=int, default=512)
    parser.add_argument("--seed", type=int, default=2026)
    parser.add_argument("--batch-size", type=int, default=64)
    parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="cpu")
    args = parser.parse_args(argv)
    if min(args.samples, args.batch_size) < 1:
        raise ValueError("Context sample and batch counts must be positive")
    evaluate_context(
        args.data,
        args.h24_checkpoint,
        args.h96_checkpoint,
        args.no_history_checkpoint,
        args.output,
        samples=args.samples,
        seed=args.seed,
        batch_size=args.batch_size,
        device=args.device,
    )


if __name__ == "__main__":
    main()