maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
5.81 kB
"""Held-out, finite-sample conformal interval calibration.
The exchangeability assumption is at the sampled-window level. Windows from one
system are correlated: disjoint systems prevent leakage but do not turn these
intervals into a system-level coverage guarantee.
"""
from __future__ import annotations
import json
from pathlib import Path
import numpy as np
VERSION = "tfm-calibration-v1"
def entry_key(field_id: int, horizon_minutes: float) -> str:
return f"{int(field_id)}:{float(horizon_minutes):g}"
def finite_quantile(scores, coverage=0.9) -> float:
scores = np.asarray(scores, dtype=float)
scores = scores[np.isfinite(scores)]
if not 0 < coverage < 1 or not len(scores):
raise ValueError("Calibration needs finite scores and coverage in (0, 1)")
rank = int(np.ceil((len(scores) + 1) * coverage))
# No finite interval can attain this finite-sample rank with too few scores.
return float("inf") if rank > len(scores) else float(np.sort(scores)[rank - 1])
def fit_calibration(
mean,
std,
target,
fields,
horizons,
mask,
*,
calibration_system_ids,
training_system_ids=(),
validation_selection_system_ids=(),
test_system_ids=(),
checkpoint_sha256=(),
normalization_version="tfm-normalization-v1",
normalization_sha256=None,
coverage=0.9,
):
ids = sorted(set(calibration_system_ids))
prohibited = set(training_system_ids) | set(validation_selection_system_ids) | set(test_system_ids)
if not ids or set(ids) & prohibited:
raise ValueError(
"Calibration systems must be nonempty and disjoint from training, selection and test"
)
mean, std, target = (np.asarray(x, dtype=float) for x in (mean, std, target))
fields, horizons, mask = np.asarray(fields), np.asarray(horizons), np.asarray(mask, dtype=bool)
if any(x.shape != mean.shape for x in (std, target, fields, horizons, mask)):
raise ValueError("Calibration arrays must have identical [sample, query] shapes")
if np.any(std[mask] <= 0) or any(np.any(~np.isfinite(x[mask])) for x in (std, mean, target)):
raise ValueError("Active predictions and targets must be finite with positive standard deviations")
scores = np.abs(target - mean) / np.maximum(std, 1e-12)
entries = {}
for field, horizon in sorted(set(zip(fields[mask].tolist(), horizons[mask].tolist()))):
selected = mask & (fields == field) & (horizons == horizon) & np.isfinite(scores)
values = scores[selected]
multiplier = finite_quantile(values, coverage)
# JSON cannot portably represent Infinity. Null means insufficient samples.
entries[entry_key(field, horizon)] = {
"multiplier": multiplier if np.isfinite(multiplier) else None,
"count": len(values),
}
return {
"version": VERSION,
"method": "absolute_residual_over_predictive_std_finite_sample_quantile",
"nominal_coverage": coverage,
"normalization_version": normalization_version,
"checkpoint_sha256": list(checkpoint_sha256),
"normalization_sha256": normalization_sha256,
"calibration_system_ids": ids,
"training_system_ids": sorted(set(training_system_ids)),
"validation_selection_system_ids": sorted(set(validation_selection_system_ids)),
"test_system_ids": sorted(set(test_system_ids)),
"entries": entries,
"limitation": "Window-level calibration; overlapping windows are correlated, no system-level guarantee.",
}
def validate_calibration(
calibration,
*,
checkpoint_sha256,
normalization_version,
normalization_sha256=None,
forbidden_system_ids=(),
):
if calibration.get("version") != VERSION:
raise ValueError("Unsupported calibration version")
if sorted(calibration.get("checkpoint_sha256", [])) != sorted(checkpoint_sha256):
raise ValueError("Calibration checkpoint fingerprints do not match")
if calibration.get("normalization_version") != normalization_version:
raise ValueError("Calibration normalization version does not match")
if normalization_sha256 is not None and calibration.get("normalization_sha256") != normalization_sha256:
raise ValueError("Calibration normalization fingerprint does not match")
ids = set(calibration.get("calibration_system_ids", []))
prohibited = set(forbidden_system_ids)
for key in ("training_system_ids", "validation_selection_system_ids", "test_system_ids"):
prohibited.update(calibration.get(key, []))
if not ids or ids & prohibited:
raise ValueError("Calibration system leakage")
coverage = calibration.get("nominal_coverage", 0)
if not 0 < coverage < 1:
raise ValueError("Calibration coverage must be in (0, 1)")
for entry in calibration.get("entries", {}).values():
multiplier = entry.get("multiplier")
if entry.get("count", 0) < 1 or (
multiplier is not None and (not np.isfinite(multiplier) or multiplier < 0)
):
raise ValueError("Invalid calibration entry count or multiplier")
return calibration
def save_calibration(calibration, path):
Path(path).write_text(json.dumps(calibration, indent=2, allow_nan=False) + "\n")
def ensemble_moments(means, stds):
"""Combine independent member distributions via the law of total variance."""
means, stds = np.asarray(means, dtype=float), np.asarray(stds, dtype=float)
if means.shape != stds.shape or means.ndim < 2 or np.any(stds < 0):
raise ValueError("Member means/stds need matching [member, ...] shapes and nonnegative stds")
mean = means.mean(axis=0)
variance = np.mean(stds**2 + means**2, axis=0) - mean**2
return mean, np.sqrt(np.maximum(variance, 0))