Update app.py

app.py

@@ -5,7 +5,6 @@ import traceback
 import time
 from datetime import datetime
 
-# 🔧 Force log output to HF container
 logging.basicConfig(
     stream=sys.stdout, level=logging.INFO,
     format="%(asctime)s [%(levelname)s] %(message)s"
@@ -13,7 +12,6 @@ logging.basicConfig(
 logger = logging.getLogger("crypto_ml")
 logger.info("🚀 Starting HF Space container...")
 
-# 1. IMPORTS
 try:
     import gradio as gr
     import yfinance as yf
@@ -27,20 +25,19 @@ except Exception as e:
     logger.error(f"❌ Import failed: {e}")
     sys.exit(1)
 
-# 2. CONFIG
 SYMBOL = "ETH-USD"
 FEE = 0.0015
-WINDOW = 30
+WINDOW = 20
 MODEL_PATH = "/tmp/rf_model.pkl"
 SCALER_PATH = "/tmp/scaler.pkl"
 
 
 # ============================================================
-# 3. SAFE DATA FETCHING
+# SAFE DATA FETCHING — Fixed MultiIndex handling
 # ============================================================
 
-def safe_download(symbol, period="60d", interval="1h"):
-    """Download with retry and robust column handling."""
+def safe_download(symbol, period="90d", interval="1h"):
+    """Download with retry and ROBUST column handling."""
     for attempt in range(3):
         try:
             logger.info(f"📥 Download attempt {attempt+1}/3: {symbol} {period} {interval}")
@@ -48,48 +45,87 @@ def safe_download(symbol, period="60d", interval="1h"):
 
             if df is None or df.empty:
                 logger.warning(f"⚠️ Empty dataframe on attempt {attempt+1}")
-                time.sleep(2)
+                time.sleep(3)
                 continue
 
-            # FIX: Handle MultiIndex columns (yfinance >= 0.2.x)
+            logger.info(f"📊 Raw shape: {df.shape}, Raw columns type: {type(df.columns)}")
+            logger.info(f"📊 Raw columns: {df.columns.tolist()}")
+
+            # ━━━ FIX: Handle MultiIndex columns properly ━━━
             if isinstance(df.columns, pd.MultiIndex):
-                logger.info("🔧 Detected MultiIndex columns, flattening...")
-                df.columns = df.columns.get_level_values(0)
+                logger.info("🔧 Detected MultiIndex columns — using droplevel(0)...")
+                # Level 0 = ticker name (e.g. "ETH-USD"), Level 1 = metric (e.g. "Open")
+                # We need Level 1 (the metric names)
+                df.columns = df.columns.droplevel(0)
+                logger.info(f"🔧 After droplevel: {df.columns.tolist()}")
 
             # Remove any duplicate columns
             df = df.loc[:, ~df.columns.duplicated()]
 
-            # Standardize column names to Title Case
-            col_map = {}
-            for c in df.columns:
-                col_map[c] = c.strip().title()
-            df.rename(columns=col_map, inplace=True)
-
-            logger.info(f"📊 Columns after cleanup: {df.columns.tolist()}")
-            logger.info(f"📊 Shape: {df.shape}")
+            # Strip whitespace and standardize to Title Case
+            df.columns = [str(c).strip().title() for c in df.columns]
+            logger.info(f"🔧 After title case: {df.columns.tolist()}")
 
-            # Validate required columns exist
+            # Validate required columns
             required = ["Open", "High", "Low", "Close", "Volume"]
             missing = [c for c in required if c not in df.columns]
             if missing:
                 logger.error(f"❌ Missing columns: {missing}. Available: {df.columns.tolist()}")
-                time.sleep(2)
-                continue
+                # Try alternate: maybe columns are lowercase
+                alt_map = {c.lower(): c for c in df.columns}
+                found_all = True
+                for req in missing:
+                    if req.lower() in alt_map:
+                        df.rename(columns={alt_map[req.lower()]: req}, inplace=True)
+                    else:
+                        found_all = False
+                
+                if not found_all:
+                    still_missing = [c for c in required if c not in df.columns]
+                    logger.error(f"❌ Still missing after alternate: {still_missing}")
+                    time.sleep(3)
+                    continue
 
             # Drop rows with NaN in required columns
             df = df.dropna(subset=required)
 
-            if len(df) < 10:
+            # Ensure numeric types
+            for col in required:
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+
+            df = df.dropna(subset=required)
+
+            if len(df) < 20:
                 logger.warning(f"⚠️ Too few rows: {len(df)}")
-                time.sleep(2)
+                time.sleep(3)
                 continue
 
-            logger.info(f"✅ Data fetched: {len(df)} rows")
+            logger.info(f"✅ Data fetched successfully: {len(df)} rows, columns: {df.columns.tolist()}")
             return df
 
         except Exception as e:
             logger.error(f"❌ Download attempt {attempt+1} failed: {e}")
-            time.sleep(2)
+            logger.error(traceback.format_exc())
+            time.sleep(3)
+
+    # Fallback: try different interval
+    logger.warning("⚠️ Trying fallback: 15m interval...")
+    try:
+        df = yf.download(symbol, period="60d", interval="15m", progress=False, threads=False)
+        if df is not None and not df.empty:
+            if isinstance(df.columns, pd.MultiIndex):
+                df.columns = df.columns.droplevel(0)
+            df.columns = [str(c).strip().title() for c in df.columns]
+            df = df.loc[:, ~df.columns.duplicated()]
+            df = df.dropna(subset=["Open", "High", "Low", "Close", "Volume"])
+            for col in ["Open", "High", "Low", "Close", "Volume"]:
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+            df = df.dropna(subset=["Open", "High", "Low", "Close", "Volume"])
+            if len(df) >= 20:
+                logger.info(f"✅ Fallback data: {len(df)} rows")
+                return df
+    except Exception as e2:
+        logger.error(f"❌ Fallback also failed: {e2}")
 
     logger.error("❌ All download attempts failed")
     return pd.DataFrame()
@@ -97,23 +133,37 @@ def safe_download(symbol, period="60d", interval="1h"):
 
 def fetch_and_prepare():
     """Fetch OHLCV data, compute indicators, build windowed features."""
-    df = safe_download(SYMBOL, period="60d", interval="1h")
+    df = safe_download(SYMBOL, period="90d", interval="1h")
 
     if df.empty:
-        raise ValueError("Gagal mengambil data dari Yahoo Finance. Coba lagi nanti.")
+        raise ValueError(
+            "Gagal mengambil data dari Yahoo Finance. "
+            "Kemungkinan koneksi bermasalah atau rate-limited. Tunggu beberapa menit lalu coba lagi."
+        )
 
     # Resample to 4h
+    logger.info(f"📊 Before resample: {len(df)} rows")
     df = df.resample("4h").agg({
         "Open": "first", "High": "max", "Low": "min",
         "Close": "last", "Volume": "sum"
     }).dropna()
 
-    if len(df) < 20:
-        raise ValueError(f"Data setelah resample terlalu sedikit: {len(df)} baris. Butuh minimal 20.")
-
-    df.columns = [c.lower() for c in df.columns]
-    df.index.name = "timestamp"
-    df = df.reset_index()
+    if len(df) < 30:
+        logger.warning(f"⚠️ Only {len(df)} rows after 4h resample, trying 1h resample...")
+        df = safe_download(SYMBOL, period="90d", interval="1h")
+        if not df.empty:
+            # Use 1h data directly
+            df.columns = [c.lower() for c in df.columns]
+            df.index.name = "timestamp"
+            df = df.reset_index()
+            logger.info(f"📊 Using 1h data: {len(df)} rows")
+            # Skip resample, continue with 1h
+        else:
+            raise ValueError(f"Data terlalu sedikit setelah resample: {len(df)} baris.")
+    else:
+        df.columns = [c.lower() for c in df.columns]
+        df.index.name = "timestamp"
+        df = df.reset_index()
 
     logger.info(f"📊 After resample: {len(df)} rows")
 
@@ -130,8 +180,7 @@ def fetch_and_prepare():
     df["bb_upper"] = sma20 + std20 * 2
     df["bb_lower"] = sma20 - std20 * 2
     bb_width = (df["bb_upper"] - df["bb_lower"]).replace(0, np.nan)
-    df["bb_pct"] = ((df["close"] - df["bb_lower"]) / bb_width).fillna(0.5)
-    df["bb_pct"] = df["bb_pct"].clip(0, 1)
+    df["bb_pct"] = ((df["close"] - df["bb_lower"]) / bb_width).fillna(0.5).clip(0, 1)
 
     # ── MACD ──
     ema12 = df["close"].ewm(span=12, adjust=False).mean()
@@ -160,7 +209,8 @@ def fetch_and_prepare():
     if len(df) < WINDOW + 5:
         raise ValueError(
             f"Data tidak cukup setelah preprocessing: {len(df)} baris. "
-            f"Butuh minimal {WINDOW + 5} baris. Coba kurangi WINDOW atau gunakan periode data lebih panjang."
+            f"Butuh minimal {WINDOW + 5} baris. "
+            f"WINDOW={WINDOW}. Coba lagi nanti saat data lebih tersedia."
         )
 
     feat_cols = [
@@ -170,23 +220,31 @@ def fetch_and_prepare():
         "ema_9", "ema_21", "vol_change", "return_1", "return_4"
     ]
 
+    # Validate all feature columns exist
+    missing_feats = [f for f in feat_cols if f not in df.columns]
+    if missing_feats:
+        raise ValueError(f"Missing feature columns: {missing_feats}. Available: {df.columns.tolist()}")
+
     X_list, y_list = [], []
     for i in range(WINDOW, len(df)):
         window_data = df.iloc[i - WINDOW:i][feat_cols].values
-        if window_data.shape[0] == WINDOW and not np.isnan(window_data).any():
+        if window_data.shape[0] == WINDOW and not np.isnan(window_data).any() and not np.isinf(window_data).any():
             X_list.append(window_data.flatten())
             y_list.append(int(df.iloc[i]["label"]))
 
     if len(X_list) < 10:
-        raise ValueError(f"Tidak cukup sampel valid setelah windowing: {len(X_list)}. Butuh minimal 10.")
+        raise ValueError(
+            f"Tidak cukup sampel valid setelah windowing: {len(X_list)}. "
+            f"Butuh minimal 10. Total rows={len(df)}, WINDOW={WINDOW}"
+        )
 
     X = np.array(X_list, dtype=np.float64)
     y = np.array(y_list, dtype=np.int32)
-
-    # Final NaN/Inf check on X
     X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
 
-    logger.info(f"✅ Features prepared: X={X.shape}, y={y.shape}, BUY={np.sum(y==1)}, SELL={np.sum(y==0)}")
+    n_buy = int(np.sum(y == 1))
+    n_sell = int(np.sum(y == 0))
+    logger.info(f"✅ Features ready: X={X.shape}, y={y.shape}, BUY={n_buy}, SELL={n_sell}")
     return X, y, df, feat_cols
 
 
@@ -219,7 +277,6 @@ def run_train():
         logger.info("🧠 Starting training...")
         X, y, df_full, feat_cols = fetch_and_prepare()
 
-        # Check class balance
         n_buy = int(np.sum(y == 1))
         n_sell = int(np.sum(y == 0))
         if n_buy < 3 or n_sell < 3:
@@ -228,18 +285,19 @@ def run_train():
         scaler = MinMaxScaler()
         X_scaled = scaler.fit_transform(X)
 
-        # Use 90% train, 10% test
-        split = max(int(len(X) * 0.90), 10)
-        if split >= len(X) - 2:
+        split = max(int(len(X) * 0.88), 10)
+        if split >= len(X) - 3:
             split = len(X) - 3
 
         X_train, X_test = X_scaled[:split], X_scaled[split:]
         y_train, y_test = y[:split], y[split:]
 
+        logger.info(f"📊 Train: {len(X_train)}, Test: {len(X_test)}")
+
         model = RandomForestClassifier(
             n_estimators=100, max_depth=8,
             min_samples_split=5, min_samples_leaf=2,
-            random_state=42, n_jobs=1,  # n_jobs=1 lebih stabil di HF
+            random_state=42, n_jobs=1,
             class_weight="balanced"
         )
         model.fit(X_train, y_train)
@@ -249,25 +307,27 @@ def run_train():
         buy_sig = int(np.sum(y_pred == 1))
         sell_sig = int(np.sum(y_pred == 0))
 
-        # Win-rate on test
         wins = int(np.sum((y_pred == 1) & (y_test == 1)))
         win_rate = (wins / max(buy_sig, 1)) * 100
 
-        # Simulated PnL
-        test_closes = df_full.iloc[split + WINDOW: split + WINDOW + len(y_test)]["close"].values
-        if len(test_closes) != len(y_pred):
-            test_closes = df_full.iloc[-len(y_pred):]["close"].values
+        # Simulated PnL using df_full close prices
+        # X_test corresponds to df_full rows [split+WINDOW : split+WINDOW+len(y_test)]
+        start_idx = split + WINDOW
+        end_idx = start_idx + len(y_test)
+        if end_idx <= len(df_full):
+            test_closes = df_full.iloc[start_idx:end_idx]["close"].values
+        else:
+            # Fallback: use last len(y_test) rows
+            test_closes = df_full.iloc[-(len(y_test) + 1):]["close"].values
 
         pnl = 0.0
-        for i in range(min(len(y_pred), len(test_closes))):
+        for i in range(min(len(y_pred), len(test_closes) - 1)):
             if y_pred[i] == 1:
-                if i < len(test_closes) - 1:
-                    ret = (test_closes[i + 1] - test_closes[i]) / test_closes[i] - FEE
-                    pnl += ret
+                ret = (test_closes[i + 1] - test_closes[i]) / test_closes[i] - FEE
+                pnl += ret
 
         pnl_pct = pnl * 100
 
-        # Feature importance top-5
         importances = model.feature_importances_
         top_n = min(5, len(feat_cols))
         top5_idx = np.argsort(importances)[-top_n:][::-1]
@@ -280,9 +340,10 @@ def run_train():
         return _html_train(acc, win_rate, pnl_pct, len(X_train),
                            len(X_test), buy_sig, sell_sig, top5, df_full)
     except Exception as e:
+        tb = traceback.format_exc()
         logger.error(f"❌ Train failed: {e}")
-        logger.error(traceback.format_exc())
-        return _html_error(str(e))
+        logger.error(tb)
+        return _html_error(str(e), tb)
 
 
 # ── Prediction ──
@@ -296,9 +357,8 @@ def run_predict():
         scaler = joblib.load(SCALER_PATH)
         model = joblib.load(MODEL_PATH)
 
-        # Get last WINDOW rows
         if len(df_full) < WINDOW:
-            return _html_warn(f"Data tidak cukup untuk prediksi: {len(df_full)} baris, butuh {WINDOW}.")
+            return _html_warn(f"Data tidak cukup: {len(df_full)} baris, butuh {WINDOW}.")
 
         latest = df_full.iloc[-WINDOW:][feat_cols].values.flatten().reshape(1, -1)
         latest = np.nan_to_num(latest, nan=0.0, posinf=0.0, neginf=0.0)
@@ -318,9 +378,10 @@ def run_predict():
         logger.info(f"🔮 {signal}  conf={confidence:.1f}%")
         return _html_signal(signal, confidence, price, rsi, macd_val, bb_pct)
     except Exception as e:
+        tb = traceback.format_exc()
         logger.error(f"❌ Predict failed: {e}")
-        logger.error(traceback.format_exc())
-        return _html_error(str(e))
+        logger.error(tb)
+        return _html_error(str(e), tb)
 
 
 def refresh_price():
@@ -328,7 +389,7 @@ def refresh_price():
 
 
 # ============================================================
-# 4. HTML BUILDERS
+# HTML BUILDERS
 # ============================================================
 
 def _html_price_header(info):
@@ -451,11 +512,12 @@ def _html_train(acc, win_rate, pnl_pct, n_tr, n_te, buys, sells, top5, df_full):
     recent = df_full.tail(n_recent)
     rows = ""
     for _, r in recent.iterrows():
-        t = r["timestamp"].strftime("%m/%d %H:%M") if pd.notna(r.get("timestamp")) else ""
-        c = float(r["close"])
+        ts_val = r.get("timestamp", None)
+        t = pd.to_datetime(ts_val).strftime("%m/%d %H:%M") if pd.notna(ts_val) else ""
+        c = float(r.get("close", 0))
         ch = float(r.get("return_1", 0)) * 100
         ch2 = "#0ECB81" if ch >= 0 else "#F6465D"
-        rv = float(r["rsi"])
+        rv = float(r.get("rsi", 50))
         rows += f"""<tr style="border-bottom:1px solid #2B3139;">
           <td style="color:#848E9C;font-size:10px;padding:5px 6px;">{t}</td>
           <td style="color:#EAECEF;font-size:10px;padding:5px 6px;text-align:right;">${c:,.2f}</td>
@@ -506,27 +568,41 @@ def _html_train(acc, win_rate, pnl_pct, n_tr, n_te, buys, sells, top5, df_full):
       </div>
       <div style="padding:8px 18px;display:flex;justify-content:space-between;">
         <span style="color:#5E6673;font-size:10px;">Train: {n_tr} · Test: {n_te}</span>
-        <span style="color:#5E6673;font-size:10px;">Split 90% / 10%</span>
+        <span style="color:#5E6673;font-size:10px;">Split 88% / 12%</span>
       </div>
     </div>"""
 
 
-def _html_error(msg):
+def _html_error(msg, tb=""):
+    # Show first 3 lines of traceback for debugging
+    tb_lines = tb.strip().split("\n")[-6:] if tb else []
+    tb_html = ""
+    if tb_lines:
+        tb_text = "<br>".join(tb_lines)
+        tb_html = f"""<details style="margin-top:10px;text-align:left;">
+          <summary style="color:#5E6673;font-size:10px;cursor:pointer;">Show Details</summary>
+          <pre style="color:#848E9C;font-size:9px;background:#2B3139;padding:8px;border-radius:4px;
+                     overflow-x:auto;margin-top:4px;white-space:pre-wrap;">{tb_text}</pre>
+        </details>"""
+
     return f"""<div style="background:#1E2329;border-radius:8px;border:1px solid rgba(246,70,93,.35);
               padding:20px;text-align:center;font-family:'Inter',sans-serif;">
       <div style="color:#F6465D;font-size:18px;font-weight:700;margin-bottom:6px;">❌ ERROR</div>
-      <div style="color:#848E9C;font-size:12px;word-break:break-word;">{msg}</div></div>"""
+      <div style="color:#848E9C;font-size:12px;word-break:break-word;">{msg}</div>
+      {tb_html}
+    </div>"""
 
 
 def _html_warn(msg):
     return f"""<div style="background:#1E2329;border-radius:8px;border:1px solid rgba(240,185,11,.35);
               padding:20px;text-align:center;font-family:'Inter',sans-serif;">
-      <div style="color:#F0B90B;font-size:18px;font-weight:700;margin-bottom:6px;">⚠️ WARNING</div>
-      <div style="color:#848E9C;font-size:12px;">{msg}</div></div>"""
+      <div style="color:#F0B90B;font-size:18px;font-weight:700;margin-bottom:6px;">⚠️</div>
+      <div style="color:#848E9C;font-size:12px;">{msg}</div>
+    </div>"""
 
 
 # ============================================================
-# 5. TRADINGVIEW WIDGET
+# TRADINGVIEW WIDGET
 # ============================================================
 
 TRADINGVIEW_HTML = """
@@ -538,7 +614,7 @@ TRADINGVIEW_HTML = """
 """
 
 # ============================================================
-# 6. CSS (BINANCE THEME)
+# CSS
 # ============================================================
 
 BINANCE_CSS = """
@@ -579,13 +655,13 @@ button {
   font-size: 13px !important;
   cursor: pointer !important;
 }
-button.primary, .btn-primary {
+button.primary {
   background: var(--accent) !important;
   color: var(--bg-primary) !important;
   border: none !important;
   box-shadow: 0 2px 8px rgba(240,185,11,.25) !important;
 }
-button.primary:hover, .btn-primary:hover {
+button.primary:hover {
   background: var(--accent-hover) !important;
   box-shadow: 0 4px 16px rgba(240,185,11,.4) !important;
   transform: translateY(-1px) !important;
@@ -616,10 +692,10 @@ input, textarea, select {
 """
 
 # ============================================================
-# 7. GRADIO UI
+# GRADIO UI
 # ============================================================
 
-logger.info("🎨 Building Gradio UI with Binance theme...")
+logger.info("🎨 Building Gradio UI...")
 
 with gr.Blocks(
     title="CryptoML Terminal — ETH/USDT",
@@ -632,7 +708,6 @@ with gr.Blocks(
     ),
 ) as demo:
 
-    # ── Navigation Bar ──
     gr.HTML("""
     <div style="display:flex;align-items:center;justify-content:space-between;
                 padding:10px 24px;background:#181A20;border-bottom:2px solid #F0B90B;
@@ -654,17 +729,13 @@ with gr.Blocks(
       </div>
     </div>""")
 
-    # ── Price Header ──
     price_header = gr.HTML(
         value=_html_price_header({"price": 0, "change_pct": 0, "high_24h": 0, "low_24h": 0, "volume": 0})
     )
 
-    # ── Chart ──
     gr.HTML(TRADINGVIEW_HTML)
 
-    # ── Main Panels ──
     with gr.Row(equal_height=True):
-        # LEFT — Controls
         with gr.Column(scale=2):
             gr.HTML("""
             <div style="padding:10px 0;border-bottom:1px solid #2B3139;margin-bottom:10px;
@@ -692,7 +763,7 @@ with gr.Blocks(
                 <span style="color:#EAECEF;font-size:10px;font-weight:500;">0.15%</span></div>
               <div style="display:flex;justify-content:space-between;margin-bottom:3px;">
                 <span style="color:#848E9C;font-size:10px;">Lookback Window</span>
-                <span style="color:#EAECEF;font-size:10px;font-weight:500;">30 bars</span></div>
+                <span style="color:#EAECEF;font-size:10px;font-weight:500;">20 bars</span></div>
               <div style="display:flex;justify-content:space-between;margin-bottom:3px;">
                 <span style="color:#848E9C;font-size:10px;">Algorithm</span>
                 <span style="color:#EAECEF;font-size:10px;font-weight:500;">RF-100</span></div>
@@ -701,18 +772,15 @@ with gr.Blocks(
                 <span style="color:#EAECEF;font-size:10px;font-weight:500;">RSI·BB·MACD·EMA</span></div>
             </div>""")
 
-        # RIGHT — Signal Output
         with gr.Column(scale=3):
             signal_output = gr.HTML(
                 value=_html_warn("Klik <b>🔮 Predict Signal</b> untuk mendapatkan sinyal.")
             )
 
-    # ── Training Results (full width) ──
     train_output = gr.HTML(
         value=_html_warn("Klik <b>⚡ Train Model</b> untuk melatih dan melihat hasil backtest.")
     )
 
-    # ── Footer ──
     gr.HTML("""
     <div style="display:flex;justify-content:space-between;align-items:center;padding:10px 24px;
                 background:#181A20;border-top:1px solid #2B3139;margin-top:12px;
@@ -723,16 +791,11 @@ with gr.Blocks(
         Data: Yahoo Finance · Chart: TradingView · ML: scikit-learn</span>
     </div>""")
 
-    # ── Events ──
     train_btn.click(fn=run_train, outputs=train_output)
     predict_btn.click(fn=run_predict, outputs=signal_output)
     refresh_btn.click(fn=refresh_price, outputs=price_header)
 
-
-# ============================================================
-# 8. LAUNCH
-# ============================================================
 demo.queue(default_concurrency_limit=1)
-logger.info("✅ Gradio app built & queued. Launching on 0.0.0.0:7860...")
+logger.info("✅ App ready. Launching...")
 
 demo.launch(server_name="0.0.0.0", server_port=7860)