"""Timestral-28M forecast engine — ZeroGPU-ready (eager CUDA pack + single GPU call). Import order: `spaces` before `torch` / `timestral` (required on ZeroGPU). """ from __future__ import annotations import spaces # MUST precede torch / timestral (ZeroGPU monkey-patch) import os import time from typing import Any import numpy as np import torch from timestral import TimestralForecaster from lab.baselines import holdout_split, last_value_forecast, seasonal_naive_forecast from lab.constants import GPU_DURATION_S, MODEL_ID, MODEL_REVISION from lab.metrics import summarize_comparison from lab.quantiles import QUANTILE_LEVELS, select_band from lab.results import BacktestResult, QuantileForecast # Optional explicit local mode: TIMESTRAL_DEVICE=cpu|cuda # - cpu: documented local CPU path (spaces.GPU is a no-op off ZeroGPU) # - On ZeroGPU (SPACES_ZERO_GPU set): cpu is a hard misconfiguration (no silent fallback) _THREAD_DEFAULT = "1" os.environ.setdefault("OMP_NUM_THREADS", _THREAD_DEFAULT) os.environ.setdefault("MKL_NUM_THREADS", _THREAD_DEFAULT) os.environ.setdefault("OPENBLAS_NUM_THREADS", _THREAD_DEFAULT) os.environ.setdefault("NUMEXPR_NUM_THREADS", _THREAD_DEFAULT) os.environ.setdefault("TORCH_NUM_THREADS", _THREAD_DEFAULT) def resolve_load_device() -> torch.device: """Resolve load/inference device via official-compatible selection (not blind hardcode). Under ZeroGPU, `torch.cuda.is_available()` is monkey-patched True so module-scope `.to("cuda")` registers automatic weight packing. Off-ZeroGPU without CUDA → CPU. Explicit `TIMESTRAL_DEVICE=cpu` is the documented local CPU mode only. """ forced = os.environ.get("TIMESTRAL_DEVICE", "").strip().lower() if forced in {"cpu", "cuda"}: return torch.device(forced) return torch.device("cuda" if torch.cuda.is_available() else "cpu") def _assert_zerogpu_device_config(device: torch.device) -> None: if os.environ.get("SPACES_ZERO_GPU") and device.type != "cuda": raise RuntimeError( "SPACES_ZERO_GPU está activo pero TIMESTRAL_DEVICE/carga apunta a CPU. " "En ZeroGPU la inferencia debe ser CUDA real (sin fallback silencioso a CPU)." ) LOAD_DEVICE = resolve_load_device() _assert_zerogpu_device_config(LOAD_DEVICE) try: torch.set_num_threads(max(1, int(os.environ.get("TORCH_NUM_THREADS", "1")))) except Exception: # noqa: BLE001 pass # Eager module-scope load via official API (`device=` + model.to for ZeroGPU packing). # Official load_model default dtype is float32 (validated) — inspect after load, do not force. forecaster = TimestralForecaster.from_pretrained( MODEL_ID, revision=MODEL_REVISION, device=LOAD_DEVICE, ) forecaster.model.to(LOAD_DEVICE) forecaster.model.eval() # No CUDA compute / warmup outside @spaces.GPU — only placement for packing. def model_param_dtype() -> torch.dtype: return next(forecaster.model.parameters()).dtype def model_param_device() -> torch.device: """Official `TimestralForecaster.device` mirrors parameter device.""" return forecaster.device def _quantiles_to_numpy(out: Any) -> tuple[np.ndarray, tuple[float, ...]]: """Detach to CPU numpy — never return CUDA tensors across the ZeroGPU boundary.""" q = out.quantiles if isinstance(q, torch.Tensor): q = q.detach().float().cpu().numpy() else: q = np.asarray(q, dtype=np.float32) if q.ndim == 4: q = q[0, 0] elif q.ndim == 3: q = q[0] levels = tuple(float(x) for x in out.quantile_levels) if len(levels) != len(QUANTILE_LEVELS): levels = QUANTILE_LEVELS return np.asarray(q, dtype=np.float64), levels def _one_forecast( context: np.ndarray, horizon: int, *, symmetric_averaging: bool, clip_nonnegative: bool, seed: int, ) -> tuple[np.ndarray, tuple[float, ...]]: context = np.asarray(context, dtype=np.float32) np.random.seed(seed) torch.manual_seed(seed) with torch.inference_mode(): if hasattr(forecaster.model, "transformer") and hasattr(forecaster.model.transformer, "rope"): try: forecaster.model.transformer.rope.rebuild_inv_freq() except Exception: # noqa: BLE001 pass out = forecaster.forecast( context, prediction_length=int(horizon), symmetric_averaging=bool(symmetric_averaging), clip_nonnegative=bool(clip_nonnegative), ) return _quantiles_to_numpy(out) def _device_evidence(t0: float) -> dict[str, Any]: """Public runtime evidence (no private user data).""" param_dev = model_param_device() evidence: dict[str, Any] = { "load_device": str(LOAD_DEVICE), "forecaster_device": str(param_dev), "dtype": str(model_param_dtype()), "model_id": MODEL_ID, "model_revision": MODEL_REVISION, "gpu_duration_budget_s": GPU_DURATION_S, "spaces_zero_gpu": bool(os.environ.get("SPACES_ZERO_GPU")), "torch_cuda_is_available": bool(torch.cuda.is_available()), } if param_dev.type == "cuda" and torch.cuda.is_available(): try: evidence["cuda_device_index"] = int(torch.cuda.current_device()) evidence["cuda_name"] = str(torch.cuda.get_device_name(param_dev)) except Exception as exc: # noqa: BLE001 evidence["cuda_query_error"] = type(exc).__name__ evidence["gpu_wall_s"] = round(time.perf_counter() - t0, 4) return evidence @spaces.GPU(duration=GPU_DURATION_S) def timestral_holdout_and_future( train_context: np.ndarray, full_context: np.ndarray, holdout: int, future_horizon: int, symmetric_averaging: bool = True, clip_nonnegative: bool = True, seed: int = 0, ) -> dict[str, Any]: """Run holdout + future forecasts in a SINGLE ZeroGPU invocation. Returns only picklable CPU objects (numpy / dict / scalars). No CUDA tensors, model objects, or locks cross the process boundary. """ t0 = time.perf_counter() param_dev = model_param_device() if os.environ.get("SPACES_ZERO_GPU"): if param_dev.type != "cuda": raise RuntimeError( "ZeroGPU: el modelo no está en CUDA durante @spaces.GPU " f"(device={param_dev}). Sin fallback silencioso a CPU." ) try: probe = torch.empty(1, device="cuda") del probe except Exception as exc: # noqa: BLE001 raise RuntimeError( "ZeroGPU: no se pudo ejecutar en CUDA real " f"({type(exc).__name__}). Sin resultado inventado ni fallback CPU." ) from exc holdout_q, levels_h = _one_forecast( train_context, int(holdout), symmetric_averaging=symmetric_averaging, clip_nonnegative=clip_nonnegative, seed=int(seed), ) future_q, levels_f = _one_forecast( full_context, int(future_horizon), symmetric_averaging=symmetric_averaging, clip_nonnegative=clip_nonnegative, seed=int(seed) + 1, ) levels = levels_h if len(levels_h) == len(QUANTILE_LEVELS) else levels_f evidence = _device_evidence(t0) print( "[timestral-lab] " f"device={evidence.get('forecaster_device')} " f"cuda_name={evidence.get('cuda_name', 'n/a')} " f"dtype={evidence.get('dtype')} " f"model_sha={MODEL_REVISION} " f"gpu_wall_s={evidence['gpu_wall_s']} " f"holdout={holdout} future={future_horizon}", flush=True, ) return { "holdout_quantiles": np.asarray(holdout_q, dtype=np.float64), "future_quantiles": np.asarray(future_q, dtype=np.float64), "levels": [float(x) for x in levels], "evidence": evidence, } def _band_from_q(q: np.ndarray, levels: tuple[float, ...] | list[float]) -> QuantileForecast: levels_t = tuple(float(x) for x in levels) lo, med, hi = select_band(q, levels_t, low=0.1, high=0.9) return QuantileForecast( median=med, q_low=lo, q_high=hi, quantiles=q, levels=levels_t, low_level=0.1, high_level=0.9, ) def run_backtest_and_forecast( values: np.ndarray, *, holdout: int, future_horizon: int, seasonality: int, symmetric_averaging: bool = True, clip_nonnegative: bool = True, seed: int = 0, ) -> BacktestResult: """CPU split/baselines/metrics around a single GPU Timestral invocation.""" values = np.asarray(values, dtype=np.float64) train, holdout_actual = holdout_split(values, holdout=holdout) lv = last_value_forecast(train, horizon=holdout) sn = seasonal_naive_forecast(train, horizon=holdout, seasonality=seasonality) gpu_out = timestral_holdout_and_future( train.astype(np.float32), values.astype(np.float32), int(holdout), int(future_horizon), bool(symmetric_averaging), bool(clip_nonnegative), int(seed), ) holdout_q = np.asarray(gpu_out["holdout_quantiles"], dtype=np.float64) future_q = np.asarray(gpu_out["future_quantiles"], dtype=np.float64) levels = tuple(float(x) for x in gpu_out["levels"]) evidence = dict(gpu_out.get("evidence") or {}) holdout_band = _band_from_q(holdout_q, levels) future_band = _band_from_q(future_q, levels) metrics = summarize_comparison( holdout_actual, { "timestral": holdout_band.median, "last_value": lv, "seasonal_naive": sn, }, train=train, seasonality=seasonality, ) return BacktestResult( train=train, holdout_actual=holdout_actual, model_holdout=holdout_band.median, last_value=lv, seasonal_naive=sn, metrics=metrics, holdout_band=holdout_band, future=future_band, evidence=evidence, )