timestral-backtest-lab / lab /forecast_engine.py
Segizu's picture
Timestral Backtest Lab (ZeroGPU)
756aa06 verified
Raw History Blame Contribute Delete
9.97 kB
"""Timestral-28M forecast engine — ZeroGPU-ready (eager CUDA pack + single GPU call).
Import order: `spaces` before `torch` / `timestral` (required on ZeroGPU).
"""
from __future__ import annotations
import spaces # MUST precede torch / timestral (ZeroGPU monkey-patch)
import os
import time
from typing import Any
import numpy as np
import torch
from timestral import TimestralForecaster
from lab.baselines import holdout_split, last_value_forecast, seasonal_naive_forecast
from lab.constants import GPU_DURATION_S, MODEL_ID, MODEL_REVISION
from lab.metrics import summarize_comparison
from lab.quantiles import QUANTILE_LEVELS, select_band
from lab.results import BacktestResult, QuantileForecast
# Optional explicit local mode: TIMESTRAL_DEVICE=cpu|cuda
# - cpu: documented local CPU path (spaces.GPU is a no-op off ZeroGPU)
# - On ZeroGPU (SPACES_ZERO_GPU set): cpu is a hard misconfiguration (no silent fallback)
_THREAD_DEFAULT = "1"
os.environ.setdefault("OMP_NUM_THREADS", _THREAD_DEFAULT)
os.environ.setdefault("MKL_NUM_THREADS", _THREAD_DEFAULT)
os.environ.setdefault("OPENBLAS_NUM_THREADS", _THREAD_DEFAULT)
os.environ.setdefault("NUMEXPR_NUM_THREADS", _THREAD_DEFAULT)
os.environ.setdefault("TORCH_NUM_THREADS", _THREAD_DEFAULT)
def resolve_load_device() -> torch.device:
"""Resolve load/inference device via official-compatible selection (not blind hardcode).
Under ZeroGPU, `torch.cuda.is_available()` is monkey-patched True so module-scope
`.to("cuda")` registers automatic weight packing. Off-ZeroGPU without CUDA → CPU.
Explicit `TIMESTRAL_DEVICE=cpu` is the documented local CPU mode only.
"""
forced = os.environ.get("TIMESTRAL_DEVICE", "").strip().lower()
if forced in {"cpu", "cuda"}:
return torch.device(forced)
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
def _assert_zerogpu_device_config(device: torch.device) -> None:
if os.environ.get("SPACES_ZERO_GPU") and device.type != "cuda":
raise RuntimeError(
"SPACES_ZERO_GPU está activo pero TIMESTRAL_DEVICE/carga apunta a CPU. "
"En ZeroGPU la inferencia debe ser CUDA real (sin fallback silencioso a CPU)."
)
LOAD_DEVICE = resolve_load_device()
_assert_zerogpu_device_config(LOAD_DEVICE)
try:
torch.set_num_threads(max(1, int(os.environ.get("TORCH_NUM_THREADS", "1"))))
except Exception: # noqa: BLE001
pass
# Eager module-scope load via official API (`device=` + model.to for ZeroGPU packing).
# Official load_model default dtype is float32 (validated) — inspect after load, do not force.
forecaster = TimestralForecaster.from_pretrained(
MODEL_ID,
revision=MODEL_REVISION,
device=LOAD_DEVICE,
)
forecaster.model.to(LOAD_DEVICE)
forecaster.model.eval()
# No CUDA compute / warmup outside @spaces.GPU — only placement for packing.
def model_param_dtype() -> torch.dtype:
return next(forecaster.model.parameters()).dtype
def model_param_device() -> torch.device:
"""Official `TimestralForecaster.device` mirrors parameter device."""
return forecaster.device
def _quantiles_to_numpy(out: Any) -> tuple[np.ndarray, tuple[float, ...]]:
"""Detach to CPU numpy — never return CUDA tensors across the ZeroGPU boundary."""
q = out.quantiles
if isinstance(q, torch.Tensor):
q = q.detach().float().cpu().numpy()
else:
q = np.asarray(q, dtype=np.float32)
if q.ndim == 4:
q = q[0, 0]
elif q.ndim == 3:
q = q[0]
levels = tuple(float(x) for x in out.quantile_levels)
if len(levels) != len(QUANTILE_LEVELS):
levels = QUANTILE_LEVELS
return np.asarray(q, dtype=np.float64), levels
def _one_forecast(
context: np.ndarray,
horizon: int,
*,
symmetric_averaging: bool,
clip_nonnegative: bool,
seed: int,
) -> tuple[np.ndarray, tuple[float, ...]]:
context = np.asarray(context, dtype=np.float32)
np.random.seed(seed)
torch.manual_seed(seed)
with torch.inference_mode():
if hasattr(forecaster.model, "transformer") and hasattr(forecaster.model.transformer, "rope"):
try:
forecaster.model.transformer.rope.rebuild_inv_freq()
except Exception: # noqa: BLE001
pass
out = forecaster.forecast(
context,
prediction_length=int(horizon),
symmetric_averaging=bool(symmetric_averaging),
clip_nonnegative=bool(clip_nonnegative),
)
return _quantiles_to_numpy(out)
def _device_evidence(t0: float) -> dict[str, Any]:
"""Public runtime evidence (no private user data)."""
param_dev = model_param_device()
evidence: dict[str, Any] = {
"load_device": str(LOAD_DEVICE),
"forecaster_device": str(param_dev),
"dtype": str(model_param_dtype()),
"model_id": MODEL_ID,
"model_revision": MODEL_REVISION,
"gpu_duration_budget_s": GPU_DURATION_S,
"spaces_zero_gpu": bool(os.environ.get("SPACES_ZERO_GPU")),
"torch_cuda_is_available": bool(torch.cuda.is_available()),
}
if param_dev.type == "cuda" and torch.cuda.is_available():
try:
evidence["cuda_device_index"] = int(torch.cuda.current_device())
evidence["cuda_name"] = str(torch.cuda.get_device_name(param_dev))
except Exception as exc: # noqa: BLE001
evidence["cuda_query_error"] = type(exc).__name__
evidence["gpu_wall_s"] = round(time.perf_counter() - t0, 4)
return evidence
@spaces.GPU(duration=GPU_DURATION_S)
def timestral_holdout_and_future(
train_context: np.ndarray,
full_context: np.ndarray,
holdout: int,
future_horizon: int,
symmetric_averaging: bool = True,
clip_nonnegative: bool = True,
seed: int = 0,
) -> dict[str, Any]:
"""Run holdout + future forecasts in a SINGLE ZeroGPU invocation.
Returns only picklable CPU objects (numpy / dict / scalars). No CUDA tensors,
model objects, or locks cross the process boundary.
"""
t0 = time.perf_counter()
param_dev = model_param_device()
if os.environ.get("SPACES_ZERO_GPU"):
if param_dev.type != "cuda":
raise RuntimeError(
"ZeroGPU: el modelo no está en CUDA durante @spaces.GPU "
f"(device={param_dev}). Sin fallback silencioso a CPU."
)
try:
probe = torch.empty(1, device="cuda")
del probe
except Exception as exc: # noqa: BLE001
raise RuntimeError(
"ZeroGPU: no se pudo ejecutar en CUDA real "
f"({type(exc).__name__}). Sin resultado inventado ni fallback CPU."
) from exc
holdout_q, levels_h = _one_forecast(
train_context,
int(holdout),
symmetric_averaging=symmetric_averaging,
clip_nonnegative=clip_nonnegative,
seed=int(seed),
)
future_q, levels_f = _one_forecast(
full_context,
int(future_horizon),
symmetric_averaging=symmetric_averaging,
clip_nonnegative=clip_nonnegative,
seed=int(seed) + 1,
)
levels = levels_h if len(levels_h) == len(QUANTILE_LEVELS) else levels_f
evidence = _device_evidence(t0)
print(
"[timestral-lab] "
f"device={evidence.get('forecaster_device')} "
f"cuda_name={evidence.get('cuda_name', 'n/a')} "
f"dtype={evidence.get('dtype')} "
f"model_sha={MODEL_REVISION} "
f"gpu_wall_s={evidence['gpu_wall_s']} "
f"holdout={holdout} future={future_horizon}",
flush=True,
)
return {
"holdout_quantiles": np.asarray(holdout_q, dtype=np.float64),
"future_quantiles": np.asarray(future_q, dtype=np.float64),
"levels": [float(x) for x in levels],
"evidence": evidence,
}
def _band_from_q(q: np.ndarray, levels: tuple[float, ...] | list[float]) -> QuantileForecast:
levels_t = tuple(float(x) for x in levels)
lo, med, hi = select_band(q, levels_t, low=0.1, high=0.9)
return QuantileForecast(
median=med,
q_low=lo,
q_high=hi,
quantiles=q,
levels=levels_t,
low_level=0.1,
high_level=0.9,
)
def run_backtest_and_forecast(
values: np.ndarray,
*,
holdout: int,
future_horizon: int,
seasonality: int,
symmetric_averaging: bool = True,
clip_nonnegative: bool = True,
seed: int = 0,
) -> BacktestResult:
"""CPU split/baselines/metrics around a single GPU Timestral invocation."""
values = np.asarray(values, dtype=np.float64)
train, holdout_actual = holdout_split(values, holdout=holdout)
lv = last_value_forecast(train, horizon=holdout)
sn = seasonal_naive_forecast(train, horizon=holdout, seasonality=seasonality)
gpu_out = timestral_holdout_and_future(
train.astype(np.float32),
values.astype(np.float32),
int(holdout),
int(future_horizon),
bool(symmetric_averaging),
bool(clip_nonnegative),
int(seed),
)
holdout_q = np.asarray(gpu_out["holdout_quantiles"], dtype=np.float64)
future_q = np.asarray(gpu_out["future_quantiles"], dtype=np.float64)
levels = tuple(float(x) for x in gpu_out["levels"])
evidence = dict(gpu_out.get("evidence") or {})
holdout_band = _band_from_q(holdout_q, levels)
future_band = _band_from_q(future_q, levels)
metrics = summarize_comparison(
holdout_actual,
{
"timestral": holdout_band.median,
"last_value": lv,
"seasonal_naive": sn,
},
train=train,
seasonality=seasonality,
)
return BacktestResult(
train=train,
holdout_actual=holdout_actual,
model_holdout=holdout_band.median,
last_value=lv,
seasonal_naive=sn,
metrics=metrics,
holdout_band=holdout_band,
future=future_band,
evidence=evidence,
)