load_reference.py

"""
=============================================================================
 load_reference.py — AMFORGE/tabicl-universal-finetuned
=============================================================================
 Reference loading script for this checkpoint.

 WHY THIS FILE EXISTS
 ---------------------
 TabICL (`FinetunedTabICLClassifier`) is an in-context learner: `.fit(X, y)`
 does two things, not one --
   1. gradient-updates the backbone weights (this is what "fine-tuning" means
      here, and it IS preserved by pickle.dump()/pickle.load()).
   2. encodes the (X, y) context you gave it into an internal representation
      used at inference time (cached as `_X_encoder_` and friends). This
      encoded context is NOT preserved across a pickle round-trip.

 Consequence: `pickle.load()` on `tabicl_finetuned.pkl` will succeed with no
 error, the object looks fully formed, but calling `.predict_proba()`
 directly will raise NotFittedError (missing `_X_encoder_`), because the
 in-context state was never restored -- only the backbone weights were.

 THE FIX
 -------
 Call `.fit()` exactly ONCE per process, right after loading, with a
 representative reference/context batch (built with the exact 60-column
 feature layout below, scaled with the shipped `feature_scaler.pkl`). This
 does not retrain the backbone -- it rebuilds the in-context encoder so
 `.predict_proba()` works.

 FEATURE LAYOUT (60 columns, RAW/unscaled, before scaler.transform)
 --------------------------------------------------------------------------
 Columns 0-48 (49 base features): for tf in [M1, M5, M15, M30, H1, H4, D1]
 (finest -> coarsest), for feat in [close, trend_strength, candle_pattern,
 volatility, market_regime, candlestick_pattern, chart_pattern]:

   close_M1, trend_strength_M1, candle_pattern_M1, volatility_M1,
   market_regime_M1, candlestick_pattern_M1, chart_pattern_M1,
   close_M5, ... chart_pattern_M5,
   close_M15, ... chart_pattern_M15,
   close_M30, ... chart_pattern_M30,
   close_H1, ... chart_pattern_H1,
   close_H4, ... chart_pattern_H4,
   close_D1, ... chart_pattern_D1

 Columns 49-59 (asset-family one-hot, appended AFTER the 49 base columns,
 in this exact order -- 11 categories, this checkpoint covers real assets
 (Gold, BTC) AND synthetic indices in a single model):
   xauusd, btcusd, boom, crash, volatility, flipx, gainx, painx, fxvol,
   sfxvol, other
 -- 1.0 for the symbol's family, 0.0 elsewhere. A symbol not recognized by
 your own family-detection logic falls into "other".

 Each *_tf block is computed on that timeframe's OWN native OHLCV bars (not
 resampled from a finer timeframe), then aligned onto the finest available
 timeframe's grid via merge_asof(direction="backward") -- never a
 not-yet-closed bar from a coarser timeframe. A timeframe not available for
 a given symbol gets its 7 columns filled with 0.0 (deliberate fallback,
 matches production behavior -- not a bug).

 Formulas (c=close, o=open, h=high, l=low, on that timeframe's own bars):
   - close: raw close price c (NOT a return / pct-change).
   - trend_strength: (MA10(c) - MA30(c)) / MA30(c), 0.0 during warm-up.
   - volatility: rolling 99-bar std of c.pct_change(), 0.0 during warm-up.
   - market_regime: 1.0 if MA10(c) > MA50(c) else -1.0; 0.0 while either MA
     is still NaN (warm-up).
   - candle_pattern (single-bar, first match wins):
       body=|c-o|, rng=h-l, upper_shadow=h-max(o,c), lower_shadow=min(o,c)-l
       1 (doji)    if rng!=0 and body/rng < 0.1
       2 (hammer)  elif lower_shadow > 2*body and upper_shadow < 0.5*body
       3 (shoot.*) elif upper_shadow > 2*body and lower_shadow < 0.5*body
       0 otherwise
   - candlestick_pattern (multi-bar, first match wins, shift(1)/shift(2)):
       1 (bull. engulfing) if c>o, prev_c<prev_o, c>prev_o, o<prev_c
       2 (evening star) elif prev2_c>prev2_o, prior bar is a doji
         (|prev_c-prev_o| < 0.1*(prev_h-prev_l)), and c<o
       3 (morning star) elif prev2_c<prev2_o, same doji condition, and c>o
       0 otherwise
   - chart_pattern (first match wins):
       1 (double top)    - two local peaks ~2 bars apart in a 5-bar window,
                            tops within 0.1% of their mean
       2 (double bottom) - same with local bottoms
       3 (head & shoulders) - 7-bar window, left shoulder/head/right
                            shoulder, shoulders within 0.1% of their mean
       0 otherwise

 Labels used during training (0=Buy, 1=Sell, 2=Hold) come from a
 triple-barrier scheme (TP=2.0xATR14, SL=1.0xATR14, horizon=60 bars of the
 finest active timeframe) -> only relevant for reproducing training labels,
 not for inference features.

 IMPORTANT: feature_scaler.pkl was fit on the FULL 60-dim vector, including
 the one-hot columns. Do not refit your own scaler, and do not scale the
 one-hot block separately -- always reuse the shipped feature_scaler.pkl.
 Training also capped samples at 1600 per symbol (balanced representation
 across families) -- irrelevant for inference, only mentioned for context.

 Copyright (c) 2026 AMEFORGE. All rights reserved.
=============================================================================
"""

import pickle
import numpy as np
from huggingface_hub import hf_hub_download

REPO_ID = "AMFORGE/tabicl-universal-finetuned"

TIMEFRAMES_ORDER = ["M1", "M5", "M15", "M30", "H1", "H4", "D1"]
BASE_FEATURE_NAMES = ["close", "trend_strength", "candle_pattern",
                      "volatility", "market_regime", "candlestick_pattern", "chart_pattern"]
FEATURE_COLUMNS = [f"{feat}_{tf}" for tf in TIMEFRAMES_ORDER for feat in BASE_FEATURE_NAMES]
ASSET_FAMILIES = ["xauusd", "btcusd", "boom", "crash", "volatility", "flipx",
                   "gainx", "painx", "fxvol", "sfxvol", "other"]


def load_checkpoint():
    """Downloads and loads the fine-tuned classifier + its feature scaler.
    Returns (clf, scaler). clf is NOT yet ready for predict_proba() at this
    point -- see rebuild_context() below."""
    ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="tabicl_finetuned.pkl")
    scaler_path = hf_hub_download(repo_id=REPO_ID, filename="feature_scaler.pkl")

    with open(ckpt_path, "rb") as f:
        clf = pickle.load(f)
    with open(scaler_path, "rb") as f:
        scaler = pickle.load(f)
    return clf, scaler


def rebuild_context(clf, scaler, X_context, y_context):
    """
    REQUIRED after every load_checkpoint() call, in every fresh process.

    X_context: np.ndarray of shape (n, 60), RAW/unscaled, built with the
               exact FEATURE_COLUMNS + ASSET_FAMILIES layout documented
               above.
    y_context: np.ndarray of shape (n,), labels in {0, 1, 2} (Buy/Sell/Hold).

    This does NOT retrain the backbone weights -- it rebuilds the in-context
    encoder (_X_encoder_) that pickle could not preserve. A slice of your
    own recent, correctly-labeled history is fine as the context set; it
    does not need to be the original training data.
    """
    X_scaled = scaler.transform(np.asarray(X_context, dtype=float))
    clf.fit(X_scaled, np.asarray(y_context))
    return clf


def predict(clf, scaler, X_new):
    """X_new: np.ndarray of shape (n, 60), RAW/unscaled, same 60-column layout.
    Returns array of shape (n, 3): [p_buy, p_sell, p_hold]."""
    X_scaled = scaler.transform(np.asarray(X_new, dtype=float))
    return clf.predict_proba(X_scaled)


if __name__ == "__main__":
    # Minimal end-to-end example. Replace X_context/y_context/X_new with
    # your own real feature vectors built from the layout documented above.
    clf, scaler = load_checkpoint()

    n_context, n_features = 500, len(FEATURE_COLUMNS) + len(ASSET_FAMILIES)
    X_context = np.random.randn(n_context, n_features)  # placeholder -- use real data
    y_context = np.random.randint(0, 3, size=n_context)  # placeholder -- use real labels

    clf = rebuild_context(clf, scaler, X_context, y_context)

    X_new = np.random.randn(3, n_features)  # placeholder -- use real data
    proba = predict(clf, scaler, X_new)
    print("Buy/Sell/Hold probabilities:\n", proba)