Update app.py

app.py

@@ -1,6 +1,6 @@
 # ══════════════════════════════════════════════════════════════════════════════
-#  EUR/USD  ·  RIDGE FORECAST ENGINE  ·  Ultra-Premium Edition
-#  Model: Stacked Ridge Regression (L1+L2 optimised, polynomial features)
+#  EUR/USD  ·  ENSEMBLE FORECAST ENGINE  ·  Ultra-Premium Edition v2
+#  Model: Ensemble Ridge + GBM · ATR-normalised target · Regime filter · Threshold
 #  Forecast horizon: 1 or 2 days ahead
 #  TP/SL: ATR-based risk management  ·  Signal scoring  ·  Backtest
 #  Data: Yahoo Finance (yfinance)
@@ -15,6 +15,8 @@ from sklearn.linear_model import Ridge, RidgeCV
 from sklearn.preprocessing import StandardScaler, PolynomialFeatures
 from sklearn.pipeline import Pipeline
 from sklearn.metrics import mean_absolute_error
+from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.model_selection import TimeSeriesSplit
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
@@ -96,6 +98,43 @@ def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series:
     return 100 - (100 / (1 + gain / (loss + 1e-10)))
 
 
+def detect_regime(df: pd.DataFrame) -> pd.Series:
+    """
+    Market regime detection:
+      +1 = TREND  (directional, EMA spread wide, ADX-proxy high)
+       0 = NEUTRAL
+      -1 = RANGE  (mean-reverting, BB squeeze, flat MAs)
+    """
+    close = df['Close']
+    ema8  = close.ewm(span=8,  adjust=False).mean()
+    ema21 = close.ewm(span=21, adjust=False).mean()
+    ema55 = close.ewm(span=55, adjust=False).mean()
+
+    # Trend strength: normalised EMA spread
+    spread  = (ema8 - ema55).abs() / (close + 1e-10)
+    spread_z = (spread - spread.rolling(40).mean()) / (spread.rolling(40).std() + 1e-10)
+
+    # Volatility regime: BB width z-score
+    mid = close.rolling(20).mean()
+    bb_w = (close.rolling(20).std() * 4) / (mid + 1e-10)
+    bb_z = (bb_w - bb_w.rolling(40).mean()) / (bb_w.rolling(40).std() + 1e-10)
+
+    regime = pd.Series(0, index=df.index)
+    regime[spread_z >  0.5] =  1   # trending
+    regime[bb_z     < -0.5] = -1   # range / squeeze
+    return regime
+
+
+def normalise_target_by_atr(y_logret: np.ndarray, atr_vals: np.ndarray) -> np.ndarray:
+    """
+    Divide log-returns by ATR to make the target volatility-scaled.
+    Model learns to predict 'how many ATRs will price move', not raw pips.
+    This prevents near-zero predictions when volatility is high.
+    """
+    atr_norm = atr_vals / (atr_vals.mean() + 1e-10)   # relative ATR
+    return y_logret / (atr_norm + 1e-10)
+
+
 def create_features(data: pd.DataFrame) -> pd.DataFrame:
     """
     Premium feature set — 55+ engineered features designed to minimise MAE
@@ -218,6 +257,16 @@ def create_features(data: pd.DataFrame) -> pd.DataFrame:
         df['Volume_Ratio'] = 1.0
         df['OBV_Signal']   = 0.0
 
+    # ── Regime features ──────────────────────────────────────────────────────
+    df['Regime']       = detect_regime(df)
+    df['Regime_Trend'] = (df['Regime'] ==  1).astype(float)
+    df['Regime_Range'] = (df['Regime'] == -1).astype(float)
+
+    # BB squeeze: width below 20th percentile of last 60 days
+    bb20 = df['BB_width_20'] if 'BB_width_20' in df.columns else (
+        df['Close'].rolling(20).std() * 4 / (df['Close'].rolling(20).mean() + 1e-10))
+    df['BB_Squeeze'] = (bb20 < bb20.rolling(60).quantile(0.20)).astype(float)
+
     # ── FIX 4: Direction Composite Signal (boosts directional accuracy) ─────
     # Encodes consensus of multiple indicators into a single direction score.
     # Positive → bullish bias, Negative → bearish bias.
@@ -299,6 +348,8 @@ FEATURE_COLS = [
     'Volume_Ratio', 'OBV_Signal',
     # Direction composite (FIX 4)
     'Direction_Score', 'Momentum_Consistency',
+    # Regime
+    'Regime', 'Regime_Trend', 'Regime_Range', 'BB_Squeeze',
 ]
 
 
@@ -311,19 +362,15 @@ FEATURE_COLS = [
 def build_ridge_pipeline(alpha: float) -> Pipeline:
     """Ridge pipeline with degree-2 interactions (no bias terms in poly)."""
     return Pipeline([
-        ('scaler', StandardScaler()),
-        ('poly',   PolynomialFeatures(degree=2, interaction_only=True,
-                                      include_bias=False)),
-        ('scaler2', StandardScaler(with_mean=False)),   # re-scale after poly
-        ('ridge',  Ridge(alpha=alpha, fit_intercept=True)),
+        ('scaler',  StandardScaler()),
+        ('poly',    PolynomialFeatures(degree=2, interaction_only=True,
+                                       include_bias=False)),
+        ('scaler2', StandardScaler(with_mean=False)),
+        ('ridge',   Ridge(alpha=alpha, fit_intercept=True)),
     ])
 
 
 def select_best_alpha(X_tr, y_tr) -> float:
-    """
-    RidgeCV finds optimal alpha from a fine logarithmic grid.
-    Evaluated on 5-fold time-series split (no shuffle).
-    """
     alphas = np.logspace(-3, 4, 60)
     scaler = StandardScaler()
     Xs = scaler.fit_transform(X_tr)
@@ -332,54 +379,115 @@ def select_best_alpha(X_tr, y_tr) -> float:
     return float(rcv.alpha_)
 
 
-def train_ridge(df: pd.DataFrame, horizon: int = 1):
+# ══════════════════════════════════════════════════════════════════════════════
+#  ENSEMBLE MODEL: Ridge + Gradient Boosting
+# ══════════════════════════════════════════════════════════════════════════════
+
+def train_ensemble(df: pd.DataFrame, horizon: int = 1):
     """
-    Trains the premium Ridge model.
-    Target: log-return over `horizon` days (direction-aware).
-    Returns pipeline, test predictions (absolute), y_test (absolute), avail_cols.
+    Four upgrades over baseline:
+      A) ATR-normalised log-return target   → prevents near-zero predictions
+      B) Regime-conditioned training         → model sees market context
+      C) GBM secondary model                 → captures nonlinear patterns
+      D) Ensemble blend (Ridge 55% + GBM 45%)
+    Final signal passes through a threshold filter: if predicted move
+    is < 0.3× ATR the signal is tagged WEAK (no trade recommended).
     """
     avail = [c for c in FEATURE_COLS if c in df.columns]
     X     = df[avail].values
     close = df['Close'].values
+    atr   = df['ATR'].values if 'ATR' in df.columns else np.full(len(close), 1e-4)
 
-    # --- FIX 1: Use log-return as target (direction-aware, scale-invariant) ---
-    # log-return = log(close[t+horizon] / close[t])
-    # This directly encodes direction: positive = UP, negative = DOWN
+    # ── A. ATR-normalised log-return target ───────────────────────────────────
     y_logret = np.log(close[horizon:] / close[:len(close) - horizon])
+    atr_aligned = atr[:len(y_logret)]
+    y_norm   = normalise_target_by_atr(y_logret, atr_aligned)   # scaled target
 
-    X = X[:len(y_logret)]
-    close_aligned = close[:len(y_logret)]
+    X = X[:len(y_norm)]
+    close_aligned = close[:len(y_norm)]
+    atr_aligned   = atr_aligned[:len(y_norm)]
 
-    split = -15   # 15-day test window for robust backtest
-    X_train, X_test       = X[:split],           X[split:]
-    y_train, y_logret_test = y_logret[:split],   y_logret[split:]
+    split = -15
+    X_train, X_test       = X[:split],         X[split:]
+    y_train, y_norm_test  = y_norm[:split],     y_norm[split:]
+    y_raw_test            = y_logret[len(y_logret)+split:]
     close_test            = close_aligned[split:]
+    atr_test              = atr_aligned[split:]
 
-    # Find best alpha on training data only
+    # ── B. Ridge on full training set ────────────────────────────────────────
     best_alpha = select_best_alpha(X_train, y_train)
+    ridge_pipe = build_ridge_pipeline(best_alpha)
+    ridge_pipe.fit(X_train, y_train)
+    ridge_pred_norm = ridge_pipe.predict(X_test)
+
+    # ── C. GBM on residuals (boosting approach) ───────────────────────────────
+    # Fit GBM on training residuals so it learns what Ridge misses
+    ridge_train_pred = ridge_pipe.predict(X_train)
+    residuals_train  = y_train - ridge_train_pred
+
+    scaler_gbm = StandardScaler()
+    X_tr_sc = scaler_gbm.fit_transform(X_train)
+    X_te_sc = scaler_gbm.transform(X_test)
+
+    gbm = GradientBoostingRegressor(
+        n_estimators=120, max_depth=3, learning_rate=0.05,
+        subsample=0.8, min_samples_leaf=5, random_state=42,
+    )
+    gbm.fit(X_tr_sc, residuals_train)
+    gbm_correction = gbm.predict(X_te_sc)
+
+    # ── D. Ensemble blend ────────────────────────────────────────────────────
+    RIDGE_W = 0.55
+    GBM_W   = 0.45
+    ensemble_norm = ridge_pred_norm * RIDGE_W + (ridge_pred_norm + gbm_correction) * GBM_W
+
+    # Convert normalised prediction back to log-return, then to price
+    atr_norm_test = atr_test / (atr_aligned.mean() + 1e-10)
+    logret_hat    = ensemble_norm * atr_norm_test
+    pred_abs      = close_test * np.exp(logret_hat)
+    y_test_abs    = close_test * np.exp(y_raw_test)
+
+    mae = mean_absolute_error(y_test_abs, pred_abs)
 
-    # Build & fit pipeline
-    pipe = build_ridge_pipeline(best_alpha)
-    pipe.fit(X_train, y_train)
+    return (ridge_pipe, gbm, scaler_gbm, atr_aligned.mean(),
+            pred_abs, y_test_abs, avail, best_alpha, mae,
+            RIDGE_W, GBM_W)
+
+
+def predict_next_ensemble(ridge_pipe, gbm, scaler_gbm, atr_mean,
+                           df: pd.DataFrame, avail: list,
+                           horizon: int = 1,
+                           ridge_w: float = 0.55,
+                           gbm_w:   float = 0.45) -> tuple:
+    """
+    Returns (pred_price, logret, signal_strength, threshold_passed).
+    signal_strength = |predicted move| / ATR  (how many ATRs away)
+    threshold_passed = True if signal_strength >= 0.30
+    """
+    cp  = float(df['Close'].iloc[-1])
+    atr = float(df['ATR'].iloc[-1]) if 'ATR' in df.columns else 1e-4
+    x   = df[avail].iloc[-1].values.reshape(1, -1)
 
-    # Test predictions → convert log-return back to absolute price
-    logret_hat = pipe.predict(X_test)
-    pred_abs   = close_test * np.exp(logret_hat)
-    y_test_abs = close_test * np.exp(y_logret_test)
+    # Ridge prediction (normalised)
+    ridge_norm = float(ridge_pipe.predict(x)[0])
 
-    mae  = mean_absolute_error(y_test_abs, pred_abs)
+    # GBM correction on residual
+    x_sc = scaler_gbm.transform(x)
+    gbm_corr = float(gbm.predict(x_sc)[0])
 
-    return pipe, pred_abs, y_test_abs, avail, best_alpha, mae
+    ensemble_norm = ridge_norm * ridge_w + (ridge_norm + gbm_corr) * gbm_w
 
+    # Denormalise
+    atr_norm  = atr / (atr_mean + 1e-10)
+    logret    = ensemble_norm * atr_norm * horizon
+    pred_price = cp * np.exp(logret)
 
-def predict_next_ridge(pipe, df: pd.DataFrame, avail: list,
-                       horizon: int = 1) -> float:
-    """Predict absolute close price horizon days ahead using log-return."""
-    cp = float(df['Close'].iloc[-1])
-    x  = df[avail].iloc[-1].values.reshape(1, -1)
-    logret = float(pipe.predict(x)[0])
-    # Scale logret by horizon for multi-day forecasts
-    return cp * np.exp(logret * horizon)
+    # ── Sinyal eşiği ─────────────────────────────────────────────────────────
+    signal_strength  = abs(pred_price - cp) / (atr + 1e-10)
+    THRESHOLD        = 0.30   # minimum 0.30 ATR hareketi gerekiyor
+    threshold_passed = signal_strength >= THRESHOLD
+
+    return pred_price, logret, signal_strength, threshold_passed
 
 
 # ══════════════════════════════════════════════════════════════════════════════
@@ -464,7 +572,8 @@ def _ax_style(ax, title="", ylabel=""):
 
 
 def build_figure(df, pred_price, horizon, tp_sl,
-                 test_pred_abs, y_test_abs, best_alpha, mae) -> plt.Figure:
+                 test_pred_abs, y_test_abs, best_alpha, mae,
+                 regime_val=0, sig_strength=0.0, thr_passed=True) -> plt.Figure:
 
     plt.rcParams.update({
         'font.family':       'monospace',
@@ -517,6 +626,15 @@ def build_figure(df, pred_price, horizon, tp_sl,
             (i - 0.25, min(o, c)), 0.5, abs(c - o) + 1e-7,
             boxstyle="square,pad=0", linewidth=0, facecolor=color, alpha=0.75, zorder=3))
 
+    # ── Regime background shading ─────────────────────────────────────────────
+    if 'Regime' in tail.columns:
+        for i, (_, row) in enumerate(tail.iterrows()):
+            rv = int(row.get('Regime', 0))
+            if rv == 1:   # trend
+                ax_price.axvspan(i-0.5, i+0.5, alpha=0.06, color=ACID,  zorder=0)
+            elif rv == -1:  # range
+                ax_price.axvspan(i-0.5, i+0.5, alpha=0.06, color=ACID3, zorder=0)
+
     # Close line overlay (phosphor glow)
     ax_price.plot(idx, tail['Close'].values, lw=1.6, color=ACID, zorder=5, alpha=0.9, label='Close')
 
@@ -545,6 +663,14 @@ def build_figure(df, pred_price, horizon, tp_sl,
         fontfamily='monospace',
     )
 
+    # Threshold / signal strength label
+    thr_color = GREEN if thr_passed else RED
+    thr_text  = f"{'✅' if thr_passed else '⛔'} Sinyal Gücü: {sig_strength:.2f}× ATR"
+    ax_price.text(0.01, 0.04, thr_text, transform=ax_price.transAxes,
+                  fontsize=7.5, color=thr_color, fontweight='bold',
+                  fontfamily='monospace', zorder=9,
+                  bbox=dict(facecolor=PANEL, edgecolor=thr_color, alpha=0.7, pad=3))
+
     ax_price.legend(fontsize=7, framealpha=0.1, labelcolor=SUBTEXT,
                     ncol=3, loc='upper left')
     ax_price.yaxis.set_major_formatter(mticker.FormatStrFormatter('%.4f'))
@@ -625,7 +751,7 @@ def build_figure(df, pred_price, horizon, tp_sl,
     dir_color = GREEN if tp_sl['direction'] == 'LONG' else WARM
 
     fig.text(0.055, 0.97,
-             "EUR/USD  ·  RIDGE FORECAST ENGINE  ·  ULTRA-PREMIUM",
+             "EUR/USD  ·  ENSEMBLE FORECAST ENGINE  ·  ULTRA-PREMIUM",
              fontsize=12, color=TEXT, fontweight='bold', va='top',
              fontfamily='monospace')
     fig.text(0.055, 0.955, dir_label,
@@ -660,15 +786,22 @@ def predict_eurusd(horizon: int = 1):
         if len(df) < 40:
             return "❌ Yetersiz veri (min 40 satır gerekli).", None
 
-        # 3. Train Ridge + backtest
-        pipe, test_pred_abs, y_test_abs, avail, best_alpha, mae = train_ridge(df, horizon)
-
-        # 4. Predict next
-        pred_price = predict_next_ridge(pipe, df, avail, horizon)
+        # 3. Train Ensemble (Ridge + GBM) + backtest
+        (ridge_pipe, gbm, scaler_gbm, atr_mean,
+         test_pred_abs, y_test_abs, avail, best_alpha, mae,
+         ridge_w, gbm_w) = train_ensemble(df, horizon)
 
-        # 5. TP/SL
+        # 4. Predict next (ensemble)
         cp  = float(df['Close'].iloc[-1])
         atr = float(df['ATR'].iloc[-1])
+        pred_price, logret_val, sig_strength, thr_passed = predict_next_ensemble(
+            ridge_pipe, gbm, scaler_gbm, atr_mean, df, avail, horizon, ridge_w, gbm_w)
+
+        # 4b. Regime
+        regime_val = int(df['Regime'].iloc[-1]) if 'Regime' in df.columns else 0
+        regime_lbl = {1: "TREND ▶", 0: "NÖTR ◆", -1: "RANGE ◀"}.get(regime_val, "?")
+
+        # 5. TP/SL
         tp_sl = compute_tp_sl(cp, pred_price, atr,
                                atr_multiplier_sl=1.5, rr_ratio=2.0)
 
@@ -677,7 +810,6 @@ def predict_eurusd(horizon: int = 1):
 
         # 7. Backtest metrics
         rmse = float(np.sqrt(np.mean((test_pred_abs - y_test_abs) ** 2)))
-        # FIX 3: Correct directional accuracy — compare pred vs entry price
         close_vals = df['Close'].values
         n_test = len(test_pred_abs)
         entry_prices = close_vals[len(close_vals) - n_test - horizon :
@@ -710,15 +842,19 @@ def predict_eurusd(horizon: int = 1):
         W   = 54
         sep = "═" * W
 
+        thr_lbl = "✅ AKTİF SİNYAL" if thr_passed else "⛔ ZAYIF SİNYAL (trade yok)"
         lines = [
             sep,
-            f"  EUR/USD  ·  RIDGE FORECAST ENGINE  ·  ULTRA-PREMIUM",
+            f"  EUR/USD  ·  ENSEMBLE FORECAST ENGINE  ·  ULTRA-PREMIUM",
             f"  {datetime.now().strftime('%d %b %Y  %H:%M:%S')}",
             sep,
             "",
             f"  {'MEVCUT FİYAT':<24}:  {cp:.5f}",
-            f"  {f'+{horizon}G TAHMİN (RIDGE)':<24}:  {pred_price:.5f}  ({chg:+.3f}%)",
+            f"  {f'+{horizon}G TAHMİN (ENSEMBLE)':<24}:  {pred_price:.5f}  ({chg:+.3f}%)",
             f"  {'YÖN':<24}:  {tp_sl['direction']}",
+            f"  {'REJİM':<24}:  {regime_lbl}",
+            f"  {'SİNYAL GÜCÜ (ATR×)':<24}:  {sig_strength:.2f}x  →  {thr_lbl}",
+            f"  {'LOG-RETURN TAHMİN':<24}:  {logret_val*100:+.4f}%",
             f"  {'OPTİMAL ALPHA (RidgeCV)':<24}:  {best_alpha:.6f}",
             "",
             f"  {'─── RİSK YÖNETİMİ (ATR × 1.5  |  R:R 1:2)':<{W-2}}",
@@ -754,6 +890,8 @@ def predict_eurusd(horizon: int = 1):
             f"  {'RMSE':<24}:  {rmse:.7f}",
             f"  {'Yön Doğruluğu':<24}:  {hit:.1f}%",
             f"  {'Feature Sayısı':<24}:  {len(avail)}",
+            f"  {'Ensemble (Ridge/GBM)':<24}:  {ridge_w*100:.0f}% / {gbm_w*100:.0f}%",
+            f"  {'Eşik (ATR×0.30)':<24}:  {'Geçti ✅' if thr_passed else 'GEÇEMEDİ ⛔'}",
             "",
             sep,
             "  ⚠  Yalnızca eğitim amaçlıdır. Finansal tavsiye değildir.",
@@ -766,6 +904,8 @@ def predict_eurusd(horizon: int = 1):
         fig = build_figure(
             df, pred_price, horizon, tp_sl,
             test_pred_abs, y_test_abs, best_alpha, mae,
+            regime_val=regime_val, sig_strength=sig_strength,
+            thr_passed=thr_passed,
         )
 
         return text, fig
@@ -882,13 +1022,13 @@ HEADER = """
     <div>
       <h1 style="margin:0 0 4px;font-size:1.45rem;font-weight:900;
                  color:#E8EDF5;letter-spacing:.06em;font-family:monospace;">
-        EUR/USD &nbsp;·&nbsp; RIDGE FORECAST ENGINE
+        EUR/USD &nbsp;·&nbsp; ENSEMBLE FORECAST ENGINE
         <span style="color:#00FFB2;font-size:.85rem;margin-left:10px;
                      vertical-align:middle;letter-spacing:.12em;">ULTRA-PREMIUM</span>
       </h1>
       <p style="margin:0;color:#3A4A5E;font-size:.75rem;font-family:monospace;
                 letter-spacing:.04em;">
-        STACKED RIDGE · POLYNOMIAL INTERACTIONS · RIDGECV ALPHA · 75+ FEATURES · ATR TP/SL
+        RIDGE+GBM ENSEMBLE · ATR-NORM TARGET · REGIME FILTER · SIGNAL THRESHOLD · 80+ FEATURES
       </p>
     </div>
   </div>
@@ -922,7 +1062,7 @@ HEADER = """
 #  GRADIO UI
 # ══════════════════════════════════════════════════════════════════════════════
 
-with gr.Blocks(css=CSS, title="EUR/USD Ridge Forecast Engine") as app:
+with gr.Blocks(css=CSS, title="EUR/USD Ensemble Forecast Engine") as app:
     gr.HTML(HEADER)
 
     with gr.Row(elem_classes="btn-row"):