""" ============================================================================= load_reference.py — AMFORGE/tabicl-universal-finetuned ============================================================================= Reference loading script for this checkpoint. WHY THIS FILE EXISTS --------------------- TabICL (`FinetunedTabICLClassifier`) is an in-context learner: `.fit(X, y)` does two things, not one -- 1. gradient-updates the backbone weights (this is what "fine-tuning" means here, and it IS preserved by pickle.dump()/pickle.load()). 2. encodes the (X, y) context you gave it into an internal representation used at inference time (cached as `_X_encoder_` and friends). This encoded context is NOT preserved across a pickle round-trip. Consequence: `pickle.load()` on `tabicl_finetuned.pkl` will succeed with no error, the object looks fully formed, but calling `.predict_proba()` directly will raise NotFittedError (missing `_X_encoder_`), because the in-context state was never restored -- only the backbone weights were. THE FIX ------- Call `.fit()` exactly ONCE per process, right after loading, with a representative reference/context batch (built with the exact 60-column feature layout below, scaled with the shipped `feature_scaler.pkl`). This does not retrain the backbone -- it rebuilds the in-context encoder so `.predict_proba()` works. FEATURE LAYOUT (60 columns, RAW/unscaled, before scaler.transform) -------------------------------------------------------------------------- Columns 0-48 (49 base features): for tf in [M1, M5, M15, M30, H1, H4, D1] (finest -> coarsest), for feat in [close, trend_strength, candle_pattern, volatility, market_regime, candlestick_pattern, chart_pattern]: close_M1, trend_strength_M1, candle_pattern_M1, volatility_M1, market_regime_M1, candlestick_pattern_M1, chart_pattern_M1, close_M5, ... chart_pattern_M5, close_M15, ... chart_pattern_M15, close_M30, ... chart_pattern_M30, close_H1, ... chart_pattern_H1, close_H4, ... chart_pattern_H4, close_D1, ... chart_pattern_D1 Columns 49-59 (asset-family one-hot, appended AFTER the 49 base columns, in this exact order -- 11 categories, this checkpoint covers real assets (Gold, BTC) AND synthetic indices in a single model): xauusd, btcusd, boom, crash, volatility, flipx, gainx, painx, fxvol, sfxvol, other -- 1.0 for the symbol's family, 0.0 elsewhere. A symbol not recognized by your own family-detection logic falls into "other". Each *_tf block is computed on that timeframe's OWN native OHLCV bars (not resampled from a finer timeframe), then aligned onto the finest available timeframe's grid via merge_asof(direction="backward") -- never a not-yet-closed bar from a coarser timeframe. A timeframe not available for a given symbol gets its 7 columns filled with 0.0 (deliberate fallback, matches production behavior -- not a bug). Formulas (c=close, o=open, h=high, l=low, on that timeframe's own bars): - close: raw close price c (NOT a return / pct-change). - trend_strength: (MA10(c) - MA30(c)) / MA30(c), 0.0 during warm-up. - volatility: rolling 99-bar std of c.pct_change(), 0.0 during warm-up. - market_regime: 1.0 if MA10(c) > MA50(c) else -1.0; 0.0 while either MA is still NaN (warm-up). - candle_pattern (single-bar, first match wins): body=|c-o|, rng=h-l, upper_shadow=h-max(o,c), lower_shadow=min(o,c)-l 1 (doji) if rng!=0 and body/rng < 0.1 2 (hammer) elif lower_shadow > 2*body and upper_shadow < 0.5*body 3 (shoot.*) elif upper_shadow > 2*body and lower_shadow < 0.5*body 0 otherwise - candlestick_pattern (multi-bar, first match wins, shift(1)/shift(2)): 1 (bull. engulfing) if c>o, prev_cprev_o, oprev2_o, prior bar is a doji (|prev_c-prev_o| < 0.1*(prev_h-prev_l)), and co 0 otherwise - chart_pattern (first match wins): 1 (double top) - two local peaks ~2 bars apart in a 5-bar window, tops within 0.1% of their mean 2 (double bottom) - same with local bottoms 3 (head & shoulders) - 7-bar window, left shoulder/head/right shoulder, shoulders within 0.1% of their mean 0 otherwise Labels used during training (0=Buy, 1=Sell, 2=Hold) come from a triple-barrier scheme (TP=2.0xATR14, SL=1.0xATR14, horizon=60 bars of the finest active timeframe) -> only relevant for reproducing training labels, not for inference features. IMPORTANT: feature_scaler.pkl was fit on the FULL 60-dim vector, including the one-hot columns. Do not refit your own scaler, and do not scale the one-hot block separately -- always reuse the shipped feature_scaler.pkl. Training also capped samples at 1600 per symbol (balanced representation across families) -- irrelevant for inference, only mentioned for context. Copyright (c) 2026 AMEFORGE. All rights reserved. ============================================================================= """ import pickle import numpy as np from huggingface_hub import hf_hub_download REPO_ID = "AMFORGE/tabicl-universal-finetuned" TIMEFRAMES_ORDER = ["M1", "M5", "M15", "M30", "H1", "H4", "D1"] BASE_FEATURE_NAMES = ["close", "trend_strength", "candle_pattern", "volatility", "market_regime", "candlestick_pattern", "chart_pattern"] FEATURE_COLUMNS = [f"{feat}_{tf}" for tf in TIMEFRAMES_ORDER for feat in BASE_FEATURE_NAMES] ASSET_FAMILIES = ["xauusd", "btcusd", "boom", "crash", "volatility", "flipx", "gainx", "painx", "fxvol", "sfxvol", "other"] def load_checkpoint(): """Downloads and loads the fine-tuned classifier + its feature scaler. Returns (clf, scaler). clf is NOT yet ready for predict_proba() at this point -- see rebuild_context() below.""" ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="tabicl_finetuned.pkl") scaler_path = hf_hub_download(repo_id=REPO_ID, filename="feature_scaler.pkl") with open(ckpt_path, "rb") as f: clf = pickle.load(f) with open(scaler_path, "rb") as f: scaler = pickle.load(f) return clf, scaler def rebuild_context(clf, scaler, X_context, y_context): """ REQUIRED after every load_checkpoint() call, in every fresh process. X_context: np.ndarray of shape (n, 60), RAW/unscaled, built with the exact FEATURE_COLUMNS + ASSET_FAMILIES layout documented above. y_context: np.ndarray of shape (n,), labels in {0, 1, 2} (Buy/Sell/Hold). This does NOT retrain the backbone weights -- it rebuilds the in-context encoder (_X_encoder_) that pickle could not preserve. A slice of your own recent, correctly-labeled history is fine as the context set; it does not need to be the original training data. """ X_scaled = scaler.transform(np.asarray(X_context, dtype=float)) clf.fit(X_scaled, np.asarray(y_context)) return clf def predict(clf, scaler, X_new): """X_new: np.ndarray of shape (n, 60), RAW/unscaled, same 60-column layout. Returns array of shape (n, 3): [p_buy, p_sell, p_hold].""" X_scaled = scaler.transform(np.asarray(X_new, dtype=float)) return clf.predict_proba(X_scaled) if __name__ == "__main__": # Minimal end-to-end example. Replace X_context/y_context/X_new with # your own real feature vectors built from the layout documented above. clf, scaler = load_checkpoint() n_context, n_features = 500, len(FEATURE_COLUMNS) + len(ASSET_FAMILIES) X_context = np.random.randn(n_context, n_features) # placeholder -- use real data y_context = np.random.randint(0, 3, size=n_context) # placeholder -- use real labels clf = rebuild_context(clf, scaler, X_context, y_context) X_new = np.random.randn(3, n_features) # placeholder -- use real data proba = predict(clf, scaler, X_new) print("Buy/Sell/Hold probabilities:\n", proba)