Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/evaluation/calibration.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 5.81 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/calibration.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/evaluation/calibration.py
-
curl -L -o calibration.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/calibration.py
5.81 kB
| """Held-out, finite-sample conformal interval calibration. | |
| The exchangeability assumption is at the sampled-window level. Windows from one | |
| system are correlated: disjoint systems prevent leakage but do not turn these | |
| intervals into a system-level coverage guarantee. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| VERSION = "tfm-calibration-v1" | |
| def entry_key(field_id: int, horizon_minutes: float) -> str: | |
| return f"{int(field_id)}:{float(horizon_minutes):g}" | |
| def finite_quantile(scores, coverage=0.9) -> float: | |
| scores = np.asarray(scores, dtype=float) | |
| scores = scores[np.isfinite(scores)] | |
| if not 0 < coverage < 1 or not len(scores): | |
| raise ValueError("Calibration needs finite scores and coverage in (0, 1)") | |
| rank = int(np.ceil((len(scores) + 1) * coverage)) | |
| # No finite interval can attain this finite-sample rank with too few scores. | |
| return float("inf") if rank > len(scores) else float(np.sort(scores)[rank - 1]) | |
| def fit_calibration( | |
| mean, | |
| std, | |
| target, | |
| fields, | |
| horizons, | |
| mask, | |
| *, | |
| calibration_system_ids, | |
| training_system_ids=(), | |
| validation_selection_system_ids=(), | |
| test_system_ids=(), | |
| checkpoint_sha256=(), | |
| normalization_version="tfm-normalization-v1", | |
| normalization_sha256=None, | |
| coverage=0.9, | |
| ): | |
| ids = sorted(set(calibration_system_ids)) | |
| prohibited = set(training_system_ids) | set(validation_selection_system_ids) | set(test_system_ids) | |
| if not ids or set(ids) & prohibited: | |
| raise ValueError( | |
| "Calibration systems must be nonempty and disjoint from training, selection and test" | |
| ) | |
| mean, std, target = (np.asarray(x, dtype=float) for x in (mean, std, target)) | |
| fields, horizons, mask = np.asarray(fields), np.asarray(horizons), np.asarray(mask, dtype=bool) | |
| if any(x.shape != mean.shape for x in (std, target, fields, horizons, mask)): | |
| raise ValueError("Calibration arrays must have identical [sample, query] shapes") | |
| if np.any(std[mask] <= 0) or any(np.any(~np.isfinite(x[mask])) for x in (std, mean, target)): | |
| raise ValueError("Active predictions and targets must be finite with positive standard deviations") | |
| scores = np.abs(target - mean) / np.maximum(std, 1e-12) | |
| entries = {} | |
| for field, horizon in sorted(set(zip(fields[mask].tolist(), horizons[mask].tolist()))): | |
| selected = mask & (fields == field) & (horizons == horizon) & np.isfinite(scores) | |
| values = scores[selected] | |
| multiplier = finite_quantile(values, coverage) | |
| # JSON cannot portably represent Infinity. Null means insufficient samples. | |
| entries[entry_key(field, horizon)] = { | |
| "multiplier": multiplier if np.isfinite(multiplier) else None, | |
| "count": len(values), | |
| } | |
| return { | |
| "version": VERSION, | |
| "method": "absolute_residual_over_predictive_std_finite_sample_quantile", | |
| "nominal_coverage": coverage, | |
| "normalization_version": normalization_version, | |
| "checkpoint_sha256": list(checkpoint_sha256), | |
| "normalization_sha256": normalization_sha256, | |
| "calibration_system_ids": ids, | |
| "training_system_ids": sorted(set(training_system_ids)), | |
| "validation_selection_system_ids": sorted(set(validation_selection_system_ids)), | |
| "test_system_ids": sorted(set(test_system_ids)), | |
| "entries": entries, | |
| "limitation": "Window-level calibration; overlapping windows are correlated, no system-level guarantee.", | |
| } | |
| def validate_calibration( | |
| calibration, | |
| *, | |
| checkpoint_sha256, | |
| normalization_version, | |
| normalization_sha256=None, | |
| forbidden_system_ids=(), | |
| ): | |
| if calibration.get("version") != VERSION: | |
| raise ValueError("Unsupported calibration version") | |
| if sorted(calibration.get("checkpoint_sha256", [])) != sorted(checkpoint_sha256): | |
| raise ValueError("Calibration checkpoint fingerprints do not match") | |
| if calibration.get("normalization_version") != normalization_version: | |
| raise ValueError("Calibration normalization version does not match") | |
| if normalization_sha256 is not None and calibration.get("normalization_sha256") != normalization_sha256: | |
| raise ValueError("Calibration normalization fingerprint does not match") | |
| ids = set(calibration.get("calibration_system_ids", [])) | |
| prohibited = set(forbidden_system_ids) | |
| for key in ("training_system_ids", "validation_selection_system_ids", "test_system_ids"): | |
| prohibited.update(calibration.get(key, [])) | |
| if not ids or ids & prohibited: | |
| raise ValueError("Calibration system leakage") | |
| coverage = calibration.get("nominal_coverage", 0) | |
| if not 0 < coverage < 1: | |
| raise ValueError("Calibration coverage must be in (0, 1)") | |
| for entry in calibration.get("entries", {}).values(): | |
| multiplier = entry.get("multiplier") | |
| if entry.get("count", 0) < 1 or ( | |
| multiplier is not None and (not np.isfinite(multiplier) or multiplier < 0) | |
| ): | |
| raise ValueError("Invalid calibration entry count or multiplier") | |
| return calibration | |
| def save_calibration(calibration, path): | |
| Path(path).write_text(json.dumps(calibration, indent=2, allow_nan=False) + "\n") | |
| def ensemble_moments(means, stds): | |
| """Combine independent member distributions via the law of total variance.""" | |
| means, stds = np.asarray(means, dtype=float), np.asarray(stds, dtype=float) | |
| if means.shape != stds.shape or means.ndim < 2 or np.any(stds < 0): | |
| raise ValueError("Member means/stds need matching [member, ...] shapes and nonnegative stds") | |
| mean = means.mean(axis=0) | |
| variance = np.mean(stds**2 + means**2, axis=0) - mean**2 | |
| return mean, np.sqrt(np.maximum(variance, 0)) | |