"""Held-out, finite-sample conformal interval calibration. The exchangeability assumption is at the sampled-window level. Windows from one system are correlated: disjoint systems prevent leakage but do not turn these intervals into a system-level coverage guarantee. """ from __future__ import annotations import json from pathlib import Path import numpy as np VERSION = "tfm-calibration-v1" def entry_key(field_id: int, horizon_minutes: float) -> str: return f"{int(field_id)}:{float(horizon_minutes):g}" def finite_quantile(scores, coverage=0.9) -> float: scores = np.asarray(scores, dtype=float) scores = scores[np.isfinite(scores)] if not 0 < coverage < 1 or not len(scores): raise ValueError("Calibration needs finite scores and coverage in (0, 1)") rank = int(np.ceil((len(scores) + 1) * coverage)) # No finite interval can attain this finite-sample rank with too few scores. return float("inf") if rank > len(scores) else float(np.sort(scores)[rank - 1]) def fit_calibration( mean, std, target, fields, horizons, mask, *, calibration_system_ids, training_system_ids=(), validation_selection_system_ids=(), test_system_ids=(), checkpoint_sha256=(), normalization_version="tfm-normalization-v1", normalization_sha256=None, coverage=0.9, ): ids = sorted(set(calibration_system_ids)) prohibited = set(training_system_ids) | set(validation_selection_system_ids) | set(test_system_ids) if not ids or set(ids) & prohibited: raise ValueError( "Calibration systems must be nonempty and disjoint from training, selection and test" ) mean, std, target = (np.asarray(x, dtype=float) for x in (mean, std, target)) fields, horizons, mask = np.asarray(fields), np.asarray(horizons), np.asarray(mask, dtype=bool) if any(x.shape != mean.shape for x in (std, target, fields, horizons, mask)): raise ValueError("Calibration arrays must have identical [sample, query] shapes") if np.any(std[mask] <= 0) or any(np.any(~np.isfinite(x[mask])) for x in (std, mean, target)): raise ValueError("Active predictions and targets must be finite with positive standard deviations") scores = np.abs(target - mean) / np.maximum(std, 1e-12) entries = {} for field, horizon in sorted(set(zip(fields[mask].tolist(), horizons[mask].tolist()))): selected = mask & (fields == field) & (horizons == horizon) & np.isfinite(scores) values = scores[selected] multiplier = finite_quantile(values, coverage) # JSON cannot portably represent Infinity. Null means insufficient samples. entries[entry_key(field, horizon)] = { "multiplier": multiplier if np.isfinite(multiplier) else None, "count": len(values), } return { "version": VERSION, "method": "absolute_residual_over_predictive_std_finite_sample_quantile", "nominal_coverage": coverage, "normalization_version": normalization_version, "checkpoint_sha256": list(checkpoint_sha256), "normalization_sha256": normalization_sha256, "calibration_system_ids": ids, "training_system_ids": sorted(set(training_system_ids)), "validation_selection_system_ids": sorted(set(validation_selection_system_ids)), "test_system_ids": sorted(set(test_system_ids)), "entries": entries, "limitation": "Window-level calibration; overlapping windows are correlated, no system-level guarantee.", } def validate_calibration( calibration, *, checkpoint_sha256, normalization_version, normalization_sha256=None, forbidden_system_ids=(), ): if calibration.get("version") != VERSION: raise ValueError("Unsupported calibration version") if sorted(calibration.get("checkpoint_sha256", [])) != sorted(checkpoint_sha256): raise ValueError("Calibration checkpoint fingerprints do not match") if calibration.get("normalization_version") != normalization_version: raise ValueError("Calibration normalization version does not match") if normalization_sha256 is not None and calibration.get("normalization_sha256") != normalization_sha256: raise ValueError("Calibration normalization fingerprint does not match") ids = set(calibration.get("calibration_system_ids", [])) prohibited = set(forbidden_system_ids) for key in ("training_system_ids", "validation_selection_system_ids", "test_system_ids"): prohibited.update(calibration.get(key, [])) if not ids or ids & prohibited: raise ValueError("Calibration system leakage") coverage = calibration.get("nominal_coverage", 0) if not 0 < coverage < 1: raise ValueError("Calibration coverage must be in (0, 1)") for entry in calibration.get("entries", {}).values(): multiplier = entry.get("multiplier") if entry.get("count", 0) < 1 or ( multiplier is not None and (not np.isfinite(multiplier) or multiplier < 0) ): raise ValueError("Invalid calibration entry count or multiplier") return calibration def save_calibration(calibration, path): Path(path).write_text(json.dumps(calibration, indent=2, allow_nan=False) + "\n") def ensemble_moments(means, stds): """Combine independent member distributions via the law of total variance.""" means, stds = np.asarray(means, dtype=float), np.asarray(stds, dtype=float) if means.shape != stds.shape or means.ndim < 2 or np.any(stds < 0): raise ValueError("Member means/stds need matching [member, ...] shapes and nonnegative stds") mean = means.mean(axis=0) variance = np.mean(stds**2 + means**2, axis=0) - mean**2 return mean, np.sqrt(np.maximum(variance, 0))