Update app.py

app.py

@@ -26,7 +26,11 @@ import pandas as pd
 import plotly.graph_objects as go
 from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
-from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import roc_auc_score
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
 
 def _launcher():
     """PATH'te uvx yoksa sırayla alternatifleri dener."""
@@ -123,61 +127,188 @@ def to_df(payload) -> pd.DataFrame:
 
 # ----------------------------------------------------------------- ML
 def make_features(df: pd.DataFrame):
-    c, h, l, v = df["close"], df["high"], df["low"], df["volume"]
+    c, h, l, o, v = df["close"], df["high"], df["low"], df["open"], df["volume"]
     f = pd.DataFrame(index=df.index)
-    for n in (1, 3, 6, 12, 24):
+    lr = np.log(c).diff()
+
+    # Getiri / momentum (çok ölçekli)
+    for n in (1, 2, 3, 6, 12, 24, 48):
         f[f"ret{n}"] = c.pct_change(n)
 
+    # Volatilite rejimi
+    for n in (6, 24, 96):
+        f[f"vol{n}"] = lr.rolling(n).std()
+    f["vol_ratio"] = f["vol6"] / (f["vol96"] + 1e-12)
+    f["vol_ratio2"] = f["vol24"] / (f["vol96"] + 1e-12)
+    f["ret24_z"] = f["ret24"] / (f["vol24"] * np.sqrt(24) + 1e-12)   # volatilite-normalize momentum
+    f["ret6_z"] = f["ret6"] / (f["vol24"] * np.sqrt(6) + 1e-12)
+
+    # RSI (iki periyot)
     d = c.diff()
-    up = d.clip(lower=0).ewm(alpha=1 / 14, adjust=False).mean()
-    dn = (-d.clip(upper=0)).ewm(alpha=1 / 14, adjust=False).mean()
-    f["rsi"] = 100 - 100 / (1 + up / (dn + 1e-12))
+    for n in (7, 14):
+        up = d.clip(lower=0).ewm(alpha=1 / n, adjust=False).mean()
+        dn = (-d.clip(upper=0)).ewm(alpha=1 / n, adjust=False).mean()
+        f[f"rsi{n}"] = 100 - 100 / (1 + up / (dn + 1e-12))
 
     macd = c.ewm(span=12, adjust=False).mean() - c.ewm(span=26, adjust=False).mean()
     f["macd_h"] = (macd - macd.ewm(span=9, adjust=False).mean()) / c
-    f["ema_dist"] = c / c.ewm(span=50, adjust=False).mean() - 1
+    for n in (20, 50, 200):
+        f[f"ema_dist{n}"] = c / c.ewm(span=n, adjust=False).mean() - 1
+    f["ema_slope"] = c.ewm(span=20, adjust=False).mean().pct_change(5)
 
     tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()], axis=1).max(axis=1)
     atr = tr.ewm(alpha=1 / 14, adjust=False).mean()
     f["atr_pct"] = atr / c
     f["hl_range"] = (h - l) / c
-    f["vol_z"] = (v - v.rolling(48).mean()) / (v.rolling(48).std() + 1e-12)
+
+    # ADX (trend gücü)
+    upm, dnm = h.diff(), -l.diff()
+    pdm = ((upm > dnm) & (upm > 0)) * upm
+    mdm = ((dnm > upm) & (dnm > 0)) * dnm
+    pdi = 100 * pdm.ewm(alpha=1 / 14, adjust=False).mean() / (atr + 1e-12)
+    mdi = 100 * mdm.ewm(alpha=1 / 14, adjust=False).mean() / (atr + 1e-12)
+    f["adx"] = (100 * (pdi - mdi).abs() / (pdi + mdi + 1e-12)).ewm(alpha=1 / 14, adjust=False).mean()
+    f["di_diff"] = pdi - mdi
+
+    # Stokastik + Bollinger + kanal konumu
+    ll, hh = l.rolling(14).min(), h.rolling(14).max()
+    f["stoch"] = (c - ll) / (hh - ll + 1e-12)
     f["bb_pos"] = (c - c.rolling(20).mean()) / (2 * c.rolling(20).std() + 1e-12)
-    return f, atr
+    f["bb_width"] = 4 * c.rolling(20).std() / c.rolling(20).mean()
+    f["dist_hi50"] = c / h.rolling(50).max() - 1
+    f["dist_lo50"] = c / l.rolling(50).min() - 1
+
+    # Mum yapısı
+    rng = (h - l) + 1e-12
+    f["body"] = (c - o) / rng
+    f["upper_wick"] = (h - np.maximum(o, c)) / rng
+    f["lower_wick"] = (np.minimum(o, c) - l) / rng
+    f["close_loc"] = (c - l) / rng
+
+    # Hacim
+    f["vol_z"] = (v - v.rolling(48).mean()) / (v.rolling(48).std() + 1e-12)
+    f["vol_trend"] = v.rolling(6).mean() / (v.rolling(48).mean() + 1e-12)
+    f["signed_vol"] = (np.sign(d) * v).rolling(12).sum() / (v.rolling(12).sum() + 1e-12)  # OBV benzeri baskı
+    f["mfi_like"] = ((c - o) / rng * v).rolling(14).sum() / (v.rolling(14).sum() + 1e-12)
+
+    # İstatistiksel (otokorelasyon, çarpıklık)
+    f["autocorr"] = lr.rolling(48).corr(lr.shift(1))
+    f["skew48"] = lr.rolling(48).skew()
+
+    # Seans / zaman (döngüsel)
+    ts = df["ts"]
+    f["hour_sin"] = np.sin(2 * np.pi * ts.dt.hour / 24)
+    f["hour_cos"] = np.cos(2 * np.pi * ts.dt.hour / 24)
+    f["dow_sin"] = np.sin(2 * np.pi * ts.dt.dayofweek / 7)
+    f["dow_cos"] = np.cos(2 * np.pi * ts.dt.dayofweek / 7)
+    return f.replace([np.inf, -np.inf], np.nan), atr
 
 
-def predict(df: pd.DataFrame, horizon: int):
+def _models():
+    return {
+        "hgb": HistGradientBoostingClassifier(max_depth=3, learning_rate=0.04, max_iter=250,
+                                              l2_regularization=2.0, min_samples_leaf=40,
+                                              random_state=42),
+        "et": ExtraTreesClassifier(n_estimators=300, max_depth=6, min_samples_leaf=30,
+                                   max_features=0.5, n_jobs=-1, random_state=42),
+        "lr": make_pipeline(StandardScaler(), LogisticRegression(C=0.05, max_iter=500)),
+    }
+
+
+def _fit_predict(Xtr, ytr, Xte, w=None):
+    """Ensemble: HGB + ExtraTrees + Logistic -> olasılık ortalaması."""
+    ps = []
+    for name, m in _models().items():
+        if name == "lr":
+            m.fit(Xtr.fillna(0), ytr, logisticregression__sample_weight=w)
+            ps.append(m.predict_proba(Xte.fillna(0))[:, 1])
+        elif name == "et":
+            m.fit(Xtr.fillna(0), ytr, sample_weight=w)
+            ps.append(m.predict_proba(Xte.fillna(0))[:, 1])
+        else:
+            m.fit(Xtr, ytr, sample_weight=w)
+            ps.append(m.predict_proba(Xte)[:, 1])
+    return np.mean(ps, axis=0)
+
+
+def _backtest(p, fwd, horizon, thr, fee):
+    """Örtüşmeyen (her `horizon` barda bir) işlemlerle net getiri istatistikleri."""
+    p, fwd = p[::horizon], fwd[::horizon]
+    pos = np.where(p > thr, 1, np.where(p < 1 - thr, -1, 0))
+    pnl = pos * fwd - np.abs(pos) * fee
+    n = int((pos != 0).sum())
+    if n < 20:
+        return {"n": n, "mean": 0.0, "t": 0.0, "hit": np.nan, "pf": np.nan}
+    tr = pnl[pos != 0]
+    gp, gl = tr[tr > 0].sum(), -tr[tr < 0].sum()
+    return {"n": n, "mean": float(tr.mean()),
+            "t": float(tr.mean() / (tr.std() / np.sqrt(n) + 1e-12)),
+            "hit": float((tr > 0).mean()), "pf": float(gp / gl) if gl > 0 else np.inf}
+
+
+def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int = 5):
     feats, atr = make_features(df)
     close = df["close"]
-    y = (close.shift(-horizon) > close).astype(float)
-    y[close.shift(-horizon).isna()] = np.nan
+    fwd = close.shift(-horizon) / close - 1
+    y = (fwd > 0).astype(float)
+    y[fwd.isna()] = np.nan
 
     data = feats.copy()
-    data["y"] = y
-    train_df = data.dropna()
-    if len(train_df) < 200:
-        raise ValueError("Yeterli veri yok (en az ~200 geçerli bar gerekir). Limit'i artır.")
-
-    X, Y = train_df.drop(columns="y"), train_df["y"].astype(int)
-    split = int(len(X) * 0.8)
-    gap = horizon  # sızıntıyı önlemek için train/test arası boşluk
-    mk = lambda: HistGradientBoostingClassifier(max_depth=3, learning_rate=0.05,
-                                                max_iter=200, l2_regularization=1.0, random_state=42)
+    data["y"], data["fwd"] = y, fwd
+    data["thr_move"] = (atr / close) * np.sqrt(horizon)
+    train_df = data.dropna(subset=list(feats.columns[feats.notna().any()]) + ["y"]).copy()
+    train_df = train_df.dropna()
+    if len(train_df) < 400:
+        raise ValueError("Yeterli veri yok (en az ~400 geçerli bar gerekir). Limit'i artır.")
+
+    cols = list(feats.columns)
+    X, Y = train_df[cols], train_df["y"].astype(int)
+    F, TM = train_df["fwd"].values, train_df["thr_move"].values
+    # Gürültü filtresi: çok küçük hareketleri (≈ yazı-tura) eğitimde ağırlıksız bırak
+    # ve büyük hareketlere daha fazla ağırlık ver.
+    W = np.clip(np.abs(F) / (TM + 1e-12), 0.05, 3.0)
+
+    # --- Purged walk-forward (expanding window) ---
+    n = len(X)
+    start = int(n * 0.4)
+    edges = np.linspace(start, n, n_folds + 1).astype(int)
+    oos_p, oos_idx = [], []
+    for k in range(n_folds):
+        tr_end, te0, te1 = edges[k] - horizon, edges[k], edges[k + 1]
+        if tr_end < 200:
+            continue
+        p = _fit_predict(X.iloc[:tr_end], Y.iloc[:tr_end], X.iloc[te0:te1], W[:tr_end])
+        oos_p.append(p)
+        oos_idx.extend(range(te0, te1))
+    oos_p = np.concatenate(oos_p)
+    oos_idx = np.array(oos_idx)
+    yo, fo = Y.values[oos_idx], F[oos_idx]
+
+    acc = float(((oos_p > 0.5).astype(int) == yo).mean())
+    base = float(max(yo.mean(), 1 - yo.mean()))
+    try:
+        auc = float(roc_auc_score(yo, oos_p))
+    except ValueError:
+        auc = 0.5
 
-    m = mk().fit(X.iloc[:split - gap], Y.iloc[:split - gap])
-    acc = float((m.predict(X.iloc[split:]) == Y.iloc[split:]).mean())
-    base = float(max(Y.iloc[split:].mean(), 1 - Y.iloc[split:].mean()))
+    # Eşik taraması (yalnızca OOS üzerinde, işlem sayısı ≥ 30 şartıyla)
+    best = {"thr": 0.55, **_backtest(oos_p, fo, horizon, 0.55, fee)}
+    for thr in (0.52, 0.54, 0.56, 0.58, 0.60, 0.62):
+        r = _backtest(oos_p, fo, horizon, thr, fee)
+        if r["n"] >= 30 and r["mean"] > best["mean"]:
+            best = {"thr": thr, **r}
 
-    final = mk().fit(X, Y)
-    last = feats.iloc[[-1]].ffill().fillna(0)
-    p_up = float(final.predict_proba(last)[0, 1])
+    # Son model: tüm veriyle, canlı tahmin
+    final_p = _fit_predict(X, Y, feats.iloc[[-1]][cols].ffill().fillna(0), W)
+    p_up = float(final_p[0])
 
     price, a = float(close.iloc[-1]), float(atr.iloc[-1])
     move = a * np.sqrt(horizon)
     center = price + (p_up - 0.5) * 2 * move
     return {
-        "p_up": p_up, "acc": acc, "baseline": base, "price": price, "atr": a,
+        "p_up": p_up, "acc": acc, "baseline": base, "auc": auc, "price": price, "atr": a,
         "center": center, "low": center - move, "high": center + move,
+        "thr": best["thr"], "bt": best, "n_oos": len(oos_p), "fee": fee,
     }
 
 
@@ -220,19 +351,23 @@ def run(symbol, timeframe, exchange, horizon, limit):
         traceback.print_exc()
         return f"**Hata:** {_root(e)}", None, ""
 
-    p = res["p_up"]
-    signal = "LONG" if p > 0.55 else "SHORT" if p < 0.45 else "NÖTR"
-    edge = "zayıf (baseline'a yakın)" if res["acc"] < res["baseline"] + 0.02 else "var"
+    p, thr, bt = res["p_up"], res["thr"], res["bt"]
+    signal = "LONG" if p > thr else "SHORT" if p < 1 - thr else "NÖTR"
+    has_edge = res["auc"] > 0.52 and bt["mean"] > 0 and bt["t"] > 1.5 and bt["n"] >= 30
+    edge = "✅ var (OOS, ücret sonrası)" if has_edge else "⚠️ kanıtlanamadı"
     md = f"""### {symbol} · {timeframe} · {int(horizon)} bar sonrası
 | | |
 |---|---|
 | Son fiyat | **{res['price']:.4f}** |
-| Yükseliş olasılığı | **{p:.1%}** → **{signal}** |
+| Yükseliş olasılığı | **{p:.1%}** → **{signal}** (eşik {thr:.2f}) |
 | Tahmini merkez | {res['center']:.4f} |
 | Aralık (ATR·√h) | {res['low']:.4f} – {res['high']:.4f} |
-| Test doğruluğu | {res['acc']:.1%} (baseline {res['baseline']:.1%}) → model edge: {edge} |
+| OOS AUC / doğruluk | {res['auc']:.3f} / {res['acc']:.1%} (baseline {res['baseline']:.1%}) |
+| OOS işlem sayısı | {bt['n']} (hit {bt['hit']:.1%}, PF {bt['pf']:.2f}) |
+| Ort. net getiri / işlem | {bt['mean']:.3%} (t = {bt['t']:.2f}, ücret {res['fee']:.2%}/işlem) |
+| Model edge | {edge} |
 
-*Eğitim/araştırma amaçlıdır; yatırım tavsiyesi değildir.*"""
+*Walk-forward (purged) out-of-sample sonuçlar. Eşik OOS'ta seçildiği için hafif iyimser olabilir. Eğitim/araştırma amaçlıdır; yatırım tavsiyesi değildir.*"""
 
     # Grafik
     tail = df.tail(150)