Update app.py

app.py

@@ -510,14 +510,17 @@ def make_features(df: pd.DataFrame):
     return f.replace([np.inf, -np.inf], np.nan), atr
 
 
-def _models():
+def _models(hp=None):
+    """hp (Optuna'dan): {"hgb": {...}, "et": {...}, "lr": {"C": ...}}; yoksa varsayılanlar."""
+    hp = hp or {}
+    h = {"max_depth": 3, "learning_rate": 0.04, "max_iter": 250, "l2_regularization": 2.0,
+         "min_samples_leaf": 40, **hp.get("hgb", {})}
+    e = {"n_estimators": 300, "max_depth": 6, "min_samples_leaf": 30, "max_features": 0.5, **hp.get("et", {})}
+    c = hp.get("lr", {}).get("C", 0.05)
     return {
-        "hgb": HistGradientBoostingClassifier(max_depth=3, learning_rate=0.04, max_iter=250,
-                                              l2_regularization=2.0, min_samples_leaf=40,
-                                              random_state=42),
-        "et": ExtraTreesClassifier(n_estimators=300, max_depth=6, min_samples_leaf=30,
-                                   max_features=0.5, n_jobs=-1, random_state=42),
-        "lr": make_pipeline(StandardScaler(), LogisticRegression(C=0.05, max_iter=500)),
+        "hgb": HistGradientBoostingClassifier(**h, random_state=42),
+        "et": ExtraTreesClassifier(**e, n_jobs=-1, random_state=42),
+        "lr": make_pipeline(StandardScaler(), LogisticRegression(C=c, max_iter=500)),
     }
 
 
@@ -537,14 +540,14 @@ def _select(Xtr, ftr, k, thr=0.95):
     return keep
 
 
-def _fit_predict(Xtr, ytr, Xte, w=None, ftr=None, k=0, with_train=False):
+def _fit_predict(Xtr, ytr, Xte, w=None, ftr=None, k=0, with_train=False, hp=None):
     """Ensemble: HGB + ExtraTrees + Logistic -> olasılık ortalaması.
     k>0: fold-içi özellik seçimi. with_train: eğitim verisi üzerindeki olasılıkları da döndür (aşırı uyum ölçümü için)."""
     if k and ftr is not None:
         sel = _select(Xtr, ftr, k)
         Xtr, Xte = Xtr[sel], Xte[sel]
     ps, pt = [], []
-    for name, m in _models().items():
+    for name, m in _models(hp).items():
         if name == "lr":
             A, B = Xtr.fillna(0), Xte.fillna(0)
             m.fit(A, ytr, logisticregression__sample_weight=w)
@@ -598,7 +601,71 @@ def _nested(p, f, sizes, horizon, fee, grid=(0.52, 0.54, 0.56, 0.58, 0.60, 0.62)
     return _stats(np.concatenate(trades) if trades else np.array([]))
 
 
-def _walk(X, Y, F, W, horizon, n_folds, k):
+class _Fixed:
+    """En iyi parametreleri aynı `suggest_*` arayüzüyle geri üretmek için."""
+    def __init__(self, params):
+        self.p = params
+
+    def suggest_int(self, name, *a, **k):
+        return self.p[name]
+
+    def suggest_float(self, name, *a, **k):
+        return self.p[name]
+
+
+def _hp_space(t):
+    return {
+        "hgb": {"max_depth": t.suggest_int("hgb_depth", 2, 5),
+                "learning_rate": t.suggest_float("hgb_lr", 0.01, 0.1, log=True),
+                "max_iter": t.suggest_int("hgb_iter", 100, 400, step=50),
+                "l2_regularization": t.suggest_float("hgb_l2", 0.5, 10.0, log=True),
+                "min_samples_leaf": t.suggest_int("hgb_leaf", 20, 200, step=20)},
+        "et": {"max_depth": t.suggest_int("et_depth", 3, 10),
+               "min_samples_leaf": t.suggest_int("et_leaf", 10, 100, step=10),
+               "max_features": t.suggest_float("et_mf", 0.2, 0.8)},
+        "lr": {"C": t.suggest_float("lr_C", 0.005, 0.5, log=True)},
+    }
+
+
+def _tune(X, Y, F, W, horizon, n_trials, timeout=240):
+    """Optuna: hiperparametre + özellik sayısı (k) ayarı. YALNIZCA geliştirme verisinin ilk %40'ında
+    (iç walk-forward, 3 dilim) yapılır; asıl walk-forward OOS ve hold-out'a hiç dokunmaz.
+    Amaç: ortalama AUC − 0.5·std (kararsız/şanslı ayarları cezalandırır)."""
+    try:
+        import optuna
+    except Exception:  # noqa: BLE001
+        return {"error": "optuna kurulu değil (pip install optuna); varsayılan ayarlar kullanıldı"}
+    m = int(len(X) * 0.4)
+    if m < 500:
+        return {"error": f"Ayar bölgesi çok küçük ({m} bar < 500); Optuna atlandı, varsayılanlar kullanıldı"}
+    Xt, Yt, Ft, Wt = X.iloc[:m], Y.iloc[:m], F[:m], W[:m]
+    edges = np.linspace(int(m * 0.4), m, 4).astype(int)
+
+    def obj(trial):
+        hp = _hp_space(trial)
+        k = trial.suggest_int("k", 10, 80, step=10)
+        aucs = []
+        for j in range(3):
+            tr_end, te0, te1 = edges[j] - horizon, edges[j], edges[j + 1]
+            if tr_end < 80:
+                continue
+            p = _fit_predict(Xt.iloc[:tr_end], Yt.iloc[:tr_end], Xt.iloc[te0:te1],
+                             Wt[:tr_end], Ft[:tr_end], k, hp=hp)
+            try:
+                aucs.append(roc_auc_score(Yt.iloc[te0:te1], p))
+            except ValueError:
+                pass
+        return float(np.mean(aucs) - 0.5 * np.std(aucs)) if aucs else 0.5
+
+    optuna.logging.set_verbosity(optuna.logging.WARNING)
+    study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42))
+    study.optimize(obj, n_trials=int(n_trials), timeout=timeout)
+    bp = study.best_params
+    return {"hp": _hp_space(_Fixed(bp)), "k": int(bp["k"]), "score": float(study.best_value),
+            "trials": len(study.trials), "region": m}
+
+
+def _walk(X, Y, F, W, horizon, n_folds, k, hp=None):
     """Purged expanding-window walk-forward. Döner: OOS olasılık, satır indeksi, fold boyutları, ort. train-test AUC farkı."""
     n = len(X)
     edges = np.linspace(int(n * 0.4), n, n_folds + 1).astype(int)
@@ -608,7 +675,7 @@ def _walk(X, Y, F, W, horizon, n_folds, k):
         if tr_end < 80:
             continue
         p, ptr = _fit_predict(X.iloc[:tr_end], Y.iloc[:tr_end], X.iloc[te0:te1],
-                              W[:tr_end], F[:tr_end], k, with_train=True)
+                              W[:tr_end], F[:tr_end], k, with_train=True, hp=hp)
         try:
             gaps.append(roc_auc_score(Y.iloc[:tr_end], ptr) - roc_auc_score(Y.iloc[te0:te1], p))
         except ValueError:
@@ -623,7 +690,7 @@ def _walk(X, Y, F, W, horizon, n_folds, k):
 
 
 def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int = 5, ext=None,
-            max_feat: int = 40):
+            max_feat: int = 40, use_optuna: bool = False, n_trials: int = 25):
     feats, atr = make_features(df)
     n_ext = 0
     if ext is not None:
@@ -663,14 +730,20 @@ def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int =
     n_dev = int(n_all * 0.8)
     Xd, Yd, Fd, Wd = X.iloc[:n_dev], Y.iloc[:n_dev], F[:n_dev], W[:n_dev]
 
-    oos_p, oos_idx, sizes, gap = _walk(Xd, Yd, Fd, Wd, horizon, n_folds, max_feat)
+    hp, tuned = None, None
+    if use_optuna:
+        tuned = _tune(Xd, Yd, Fd, Wd, horizon, n_trials)
+        if "error" not in tuned:
+            hp, max_feat = tuned["hp"], tuned["k"]
+
+    oos_p, oos_idx, sizes, gap = _walk(Xd, Yd, Fd, Wd, horizon, n_folds, max_feat, hp)
     yo, fo = Yd.values[oos_idx], Fd[oos_idx]
 
     # Teşhis (ablation): yalnızca temel (fiyat/hacim) özelliklerle aynı test
     base_cols = [c for c in cols if not c.startswith(("ts_", "dv_"))]
     auc_base = None
     if len(base_cols) < len(cols):
-        pb, ib, _, _ = _walk(Xd[base_cols], Yd, Fd, Wd, horizon, n_folds, max_feat)
+        pb, ib, _, _ = _walk(Xd[base_cols], Yd, Fd, Wd, horizon, n_folds, max_feat, hp)
         try:
             auc_base = float(roc_auc_score(Yd.values[ib], pb))
         except ValueError:
@@ -701,7 +774,7 @@ def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int =
     ho = None
     tr_end = n_dev - horizon
     if tr_end >= 200 and n_all - n_dev >= 100:
-        ph = _fit_predict(X.iloc[:tr_end], Y.iloc[:tr_end], X.iloc[n_dev:], W[:tr_end], F[:tr_end], max_feat)
+        ph = _fit_predict(X.iloc[:tr_end], Y.iloc[:tr_end], X.iloc[n_dev:], W[:tr_end], F[:tr_end], max_feat, hp=hp)
         try:
             auc_h = float(roc_auc_score(Y.values[n_dev:], ph))
         except ValueError:
@@ -710,7 +783,7 @@ def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int =
               **_stats(_trades(ph, F[n_dev:], horizon, best["thr"], fee))}
 
     # Son model: tüm veriyle, canlı tahmin
-    final_p = _fit_predict(X, Y, feats.iloc[[-1]][cols].ffill().fillna(0), W, F, max_feat)
+    final_p = _fit_predict(X, Y, feats.iloc[[-1]][cols].ffill().fillna(0), W, F, max_feat, hp=hp)
     p_up = float(final_p[0])
 
     price, a = float(close.iloc[-1]), float(atr.iloc[-1])
@@ -721,7 +794,7 @@ def predict(df: pd.DataFrame, horizon: int, fee: float = 0.0006, n_folds: int =
         "center": center, "low": center - move, "high": center + move,
         "thr": best["thr"], "bt": best, "n_oos": len(oos_p), "fee": fee,
         "n_ext": n_ext, "n_feat": len(cols), "auc_base": auc_base, "max_feat": max_feat,
-        "gap": gap, "auc_p": auc_p, "nested": nested, "holdout": ho,
+        "gap": gap, "auc_p": auc_p, "nested": nested, "holdout": ho, "tuned": tuned,
     }
 
 
@@ -772,7 +845,7 @@ def _root(e):
     return f"{type(e).__name__}: {e}"
 
 
-def run(symbol, timeframe, exchange, horizon, limit, use_deriv=True, max_feat=40):
+def run(symbol, timeframe, exchange, horizon, limit, use_deriv=True, max_feat=40, use_optuna=False, n_trials=25):
     try:
         ohlcv, deep, ind, extra = asyncio.run(run_async(symbol, timeframe, exchange, horizon, limit))
         df = to_df(ohlcv)
@@ -783,7 +856,8 @@ def run(symbol, timeframe, exchange, horizon, limit, use_deriv=True, max_feat=40
         ext = indicator_features(ind, df)
         if dfeat is not None:
             ext = dfeat if ext is None else pd.concat([ext, dfeat], axis=1)
-        res = predict(df, int(horizon), ext=ext, max_feat=int(max_feat))
+        res = predict(df, int(horizon), ext=ext, max_feat=int(max_feat),
+                      use_optuna=bool(use_optuna), n_trials=int(n_trials))
     except BaseException as e:  # noqa: BLE001
         import traceback
         traceback.print_exc()
@@ -799,6 +873,16 @@ def run(symbol, timeframe, exchange, horizon, limit, use_deriv=True, max_feat=40
         ("Hold-out AUC > 0.50", ho is not None and ho["auc"] > 0.50),
         ("Hold-out net getiri > 0 (≥20 işlem)", ho is not None and ho["n"] >= 20 and ho["mean"] > 0),
     ]
+    t = res["tuned"]
+    if t is None:
+        opt_txt = "kapalı (varsayılan hiperparametreler)"
+    elif "error" in t:
+        opt_txt = t["error"]
+    else:
+        g, e = t["hp"]["hgb"], t["hp"]["et"]
+        opt_txt = (f"{t['trials']} deneme · iç skor {t['score']:.3f} · k={t['k']} · "
+                   f"HGB derinlik {g['max_depth']}, lr {g['learning_rate']:.3f} · ET derinlik {e['max_depth']} · "
+                   f"LR C={t['hp']['lr']['C']:.3f} (yalnızca ilk %40'ta ayarlandı)")
     passed = sum(ok for _, ok in checks)
     edge = f"✅ {passed}/5 kriter" if passed >= 4 else f"⚠️ {passed}/5 kriter — kanıtlanamadı"
     checks_md = "**Aşırı uyum / güvenilirlik göstergesi**\n\n" + "\n".join(
@@ -817,6 +901,7 @@ def run(symbol, timeframe, exchange, horizon, limit, use_deriv=True, max_feat=40
 | Ort. net getiri / işlem | {bt['mean']:.3%} (t = {bt['t']:.2f}, ücret {res['fee']:.2%}/işlem) |
 | Özellik sayısı | {res['n_feat']} (Tickscope/türev: {res['n_ext']}) → fold başına en fazla {res['max_feat']} seçilir |
 | AUC yalnızca temel özelliklerle | {('%.3f' % res['auc_base']) if res['auc_base'] is not None else '—'} (ek veri kaynakları yardım ediyor mu?) |
+| Optuna | {opt_txt} |
 | Train–test AUC farkı (aşırı uyum) | {res['gap']:.3f} |
 | AUC anlamlılığı (tek yönlü p) | {res['auc_p']:.3f} |
 | Nested (dürüst) net getiri / işlem | {nest['mean']:.3%} (t = {nest['t']:.2f}, {nest['n']} işlem) |
@@ -856,12 +941,16 @@ with gr.Blocks(title="Tickscope Tahmin") as demo:
         limit = gr.Slider(300, 20000, value=5000, step=100, label="Mum sayısı (1000 üstü sayfalanır)")
     use_deriv = gr.Checkbox(True, label="Türev verisi (LiquidVision MCP: funding/OI) — mumları geçmişin kapsadığı aralığa kırpabilir")
     max_feat = gr.Slider(10, 150, value=40, step=5, label="Maks. özellik (fold içinde seçilir; az = daha az aşırı uyum)")
+    with gr.Row():
+        use_optuna = gr.Checkbox(False, label="Optuna ile hiperparametre ayarı (OOS/hold-out'a dokunmaz; yavaş)")
+        n_trials = gr.Slider(10, 100, value=25, step=5, label="Optuna deneme sayısı")
     btn = gr.Button("Tahmin et", variant="primary")
     out_md = gr.Markdown()
     out_plot = gr.Plot()
     with gr.Accordion("Tickscope ham çıktı (deep_analyze / indicators)", open=False):
         out_raw = gr.Code(language="json")
-    btn.click(run, [symbol, timeframe, exchange, horizon, limit, use_deriv, max_feat], [out_md, out_plot, out_raw])
+    btn.click(run, [symbol, timeframe, exchange, horizon, limit, use_deriv, max_feat, use_optuna, n_trials], [out_md, out_plot, out_raw])
 
 if __name__ == "__main__":
     demo.launch()
+