GoshawkVortexAI commited on
Commit
70ee01c
·
verified ·
1 Parent(s): 31a9a5d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +194 -86
app.py CHANGED
@@ -95,13 +95,14 @@ os.makedirs(MODEL_DIR, exist_ok=True)
95
  MODEL_PATH = os.path.join(MODEL_DIR, "gos_hawk_model.pkl")
96
 
97
  MIN_TRAIN_SAMPLES = 150
98
- REQUEST_TIMEOUT = 30
99
  MAX_RETRIES = 3
100
  RETRY_BACKOFF = 2
101
- HISTORY_TIME_SCALE = 6
102
 
103
- # KRİTİK: Hedef ölçeği — log getiri * 10000 = baz puan
104
- # Bu, modelin 0.1-1 aralığında öğrenmesini sağlar → stabil
 
 
105
  TARGET_SCALE = 10000.0
106
 
107
  # Tahmin clip (log getiri, ±%15)
@@ -111,6 +112,26 @@ MIN_LOG_RETURN = -0.15
111
  # Belirsizlik eşiği (log getiri)
112
  LOW_CONFIDENCE_THRESHOLD = 0.01
113
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  # ---------------------------------------------------------------------------
115
  # PUBLIC API ÇAĞRILARI
116
  # ---------------------------------------------------------------------------
@@ -118,7 +139,7 @@ LOW_CONFIDENCE_THRESHOLD = 0.01
118
 
119
  def safe_request(url: str, params: Optional[dict] = None) -> Optional[list]:
120
  headers = {
121
- "User-Agent": "GOSHAWK-ALBION-MARKET-AI/1.3 (public-api-client)",
122
  "Accept": "application/json",
123
  }
124
  for attempt in range(MAX_RETRIES):
@@ -168,9 +189,9 @@ def fetch_current_prices(item_ids: List[str], cities: List[str], qualities: str
168
  return df
169
 
170
 
171
- def fetch_history(item_ids: List[str], cities: List[str], time_scale: int = 6, qualities: str = "1") -> pd.DataFrame:
172
  all_records = []
173
- chunk_size = 30
174
  for i in range(0, len(item_ids), chunk_size):
175
  chunk = item_ids[i:i+chunk_size]
176
  url = f"{API_BASE}/stats/history/{','.join(chunk)}.json"
@@ -188,7 +209,7 @@ def fetch_history(item_ids: List[str], cities: List[str], time_scale: int = 6, q
188
  "avg_price": point.get("avg_price"),
189
  "item_count": point.get("item_count", 0),
190
  })
191
- time.sleep(0.4)
192
  if not all_records:
193
  return pd.DataFrame()
194
 
@@ -215,6 +236,7 @@ def clean_market_data(df: pd.DataFrame) -> pd.DataFrame:
215
  df["avg_price"] = pd.to_numeric(df["avg_price"], errors="coerce").astype("float64")
216
  df["item_count"] = pd.to_numeric(df["item_count"], errors="coerce").fillna(0).astype("float64")
217
  df = df[df["avg_price"] > 0].copy()
 
218
 
219
  for item in df["item_id"].unique():
220
  mask = df["item_id"] == item
@@ -254,6 +276,15 @@ def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series:
254
 
255
 
256
  def build_features(df: pd.DataFrame) -> pd.DataFrame:
 
 
 
 
 
 
 
 
 
257
  if df.empty:
258
  return df
259
 
@@ -267,10 +298,12 @@ def build_features(df: pd.DataFrame) -> pd.DataFrame:
267
 
268
  group_cols = ["item_id", "city"]
269
 
 
270
  for lag in [1, 3, 6, 12, 24, 48]:
271
  df[f"lag_{lag}h"] = df.groupby(group_cols)["avg_price"].shift(lag)
272
 
273
  df["log_price"] = np.log(df["avg_price"].replace(0, np.nan))
 
274
  for chg in [1, 6, 12, 24]:
275
  df[f"chg_{chg}h"] = df.groupby(group_cols)["log_price"].shift(chg)
276
 
@@ -281,11 +314,9 @@ def build_features(df: pd.DataFrame) -> pd.DataFrame:
281
  )
282
 
283
  df["rsi_14"] = df.groupby(group_cols)["avg_price"].transform(lambda x: compute_rsi(x, 14))
284
-
285
  df["volatility_24h"] = df.groupby(group_cols)["log_price"].transform(
286
  lambda x: x.rolling(24, min_periods=6).std()
287
  )
288
-
289
  df["volume_ma_24h"] = df.groupby(group_cols)["item_count"].transform(
290
  lambda x: x.rolling(24, min_periods=6).mean()
291
  )
@@ -294,8 +325,56 @@ def build_features(df: pd.DataFrame) -> pd.DataFrame:
294
  city_avg = df.groupby(["item_id", "timestamp"])["avg_price"].transform("mean")
295
  df["city_price_deviation"] = (df["avg_price"] - city_avg) / city_avg.replace(0, np.nan)
296
 
297
- # KRİTİK: hedef ölçeği = log getiri * 10000 (baz puan)
298
- df["target_return_24h"] = df.groupby(group_cols)["log_price"].shift(-4) * TARGET_SCALE
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
299
 
300
  return df
301
 
@@ -304,23 +383,8 @@ def build_features(df: pd.DataFrame) -> pd.DataFrame:
304
  # MODEL EĞİTİMİ
305
  # ---------------------------------------------------------------------------
306
 
307
- FEATURE_COLS = [
308
- "hour", "dayofweek", "is_weekend",
309
- "lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
310
- "chg_1h", "chg_6h", "chg_12h", "chg_24h",
311
- "ma_6h", "ma_12h", "ma_24h", "ma_48h",
312
- "rsi_14", "volatility_24h",
313
- "volume_ma_24h", "volume_ratio",
314
- "city_price_deviation",
315
- "item_code", "city_code",
316
- ]
317
-
318
 
319
  def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame]:
320
- """
321
- Feature engineering çıktısını modele hazırlar.
322
- KRİTİK: Fiyat-benzeri feature'lar log1p ile normalize edilir.
323
- """
324
  if featured_df.empty:
325
  return pd.DataFrame(), pd.Series(dtype=float), pd.DataFrame()
326
 
@@ -328,14 +392,10 @@ def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Seri
328
  df["item_code"] = df["item_id"].astype("category").cat.codes
329
  df["city_code"] = df["city"].astype("category").cat.codes
330
 
331
- df = df.dropna(subset=["target_return_24h", "lag_1h", "lag_24h"]).copy()
332
 
333
- # KRİTİK: Fiyat-benzeri feature'ları log ölçeğine çevir (binlerce silver → ~7-10)
334
- price_like_cols = [
335
- "lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
336
- "ma_6h", "ma_12h", "ma_24h", "ma_48h",
337
- ]
338
- for col in price_like_cols:
339
  if col in df.columns:
340
  df[col] = np.log1p(df[col].clip(lower=1.0))
341
 
@@ -347,6 +407,10 @@ def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Seri
347
  if "ma_24h" in df.columns:
348
  df["ma_24h"] = df["ma_24h"].fillna(df["lag_24h"])
349
 
 
 
 
 
350
  for chg in ["chg_1h", "chg_6h", "chg_12h", "chg_24h"]:
351
  if chg in df.columns:
352
  df[chg] = df[chg].fillna(0.0)
@@ -359,7 +423,7 @@ def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Seri
359
 
360
  model_df = df.dropna(subset=FEATURE_COLS + ["target_return_24h"]).copy()
361
 
362
- # KRİTİK: Target zaten baz puan (×10000). Şimdi ±1500 bp (≈ ±0.15 log) sınırı
363
  target_min = MIN_LOG_RETURN * TARGET_SCALE
364
  target_max = MAX_LOG_RETURN * TARGET_SCALE
365
  model_df["target_return_24h"] = model_df["target_return_24h"].clip(target_min, target_max)
@@ -377,17 +441,18 @@ def train_global_model(X: pd.DataFrame, y: pd.Series) -> Optional[lgb.LGBMRegres
377
  print(f"Yetersiz veri: {len(X)} < {MIN_TRAIN_SAMPLES}")
378
  return None
379
 
380
- # Hedef baz puan ölçeğinde → standart sapma ~100-300 bp
381
- y_std = y.std()
382
- if y_std < 1e-3:
383
- print(f"⚠️ Hedef standart sapması çok düşük: {y_std}. Model öğrenemez.")
 
 
384
  return None
385
 
386
  split_idx = int(len(X) * 0.8)
387
  X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
388
  y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
389
 
390
- # Regularizasyon DENGELİ — overfit değil, underfit de değil
391
  model = lgb.LGBMRegressor(
392
  objective="regression",
393
  metric="mae",
@@ -419,18 +484,14 @@ def train_global_model(X: pd.DataFrame, y: pd.Series) -> Optional[lgb.LGBMRegres
419
  callbacks=callbacks,
420
  )
421
 
422
- # Öğrendi mi kontrolü
423
  val_preds = model.predict(X_val)
424
- if np.std(val_preds) < 1e-3:
425
- print(f"⚠️ Validation tahminleri neredeyse sabit (std={np.std(val_preds):.6f}). Model öğrenmedi.")
426
 
427
  return model
428
 
429
 
430
  def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame) -> pd.DataFrame:
431
- """
432
- Ham tahmin baz puan cinsinden → log getiriye çevir → clip → exp.
433
- """
434
  if latest_features.empty or model is None:
435
  return pd.DataFrame()
436
 
@@ -439,13 +500,9 @@ def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame)
439
  print(f"Eksik özellikler: {missing}")
440
  return pd.DataFrame()
441
 
442
- # KRİTİK: predict öncesi de aynı log1p dönüşümü uygula
443
  X_pred = latest_features.copy()
444
- price_like_cols = [
445
- "lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
446
- "ma_6h", "ma_12h", "ma_24h", "ma_48h",
447
- ]
448
- for col in price_like_cols:
449
  if col in X_pred.columns:
450
  X_pred[col] = np.log1p(X_pred[col].clip(lower=1.0))
451
 
@@ -463,17 +520,14 @@ def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame)
463
  if X_ready.empty:
464
  return pd.DataFrame()
465
 
466
- raw_preds_bp = model.predict(X_ready) # baz puan
467
- raw_preds_log = raw_preds_bp / TARGET_SCALE # log getiri
468
 
469
- # Tahmin çeşitliliği kontrolü
470
- pred_std = np.std(raw_preds_log)
471
  if pred_std < 1e-5:
472
  print(f"⚠️ UYARI: Tüm tahminler aynı (std={pred_std:.8f}).")
473
 
474
- # Aşırı değerleri clip'le
475
  clipped_log = np.clip(raw_preds_log, MIN_LOG_RETURN, MAX_LOG_RETURN)
476
-
477
  pred_return_pct = (np.exp(clipped_log) - 1) * 100
478
 
479
  result = X_pred.loc[X_ready.index, ["item_id", "city", "avg_price", "item_count",
@@ -499,7 +553,7 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
499
 
500
  df = featured_df.sort_values("timestamp").copy()
501
  unique_times = df["timestamp"].sort_values().unique()
502
- if len(unique_times) < 50:
503
  return {"error": "Yeterli zaman noktası yok."}
504
 
505
  test_size = len(unique_times) // (n_splits + 1)
@@ -507,11 +561,11 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
507
  all_predictions = []
508
 
509
  for i in range(n_splits):
510
- test_start_idx = test_size * (i + 1) + 4
511
  test_end_idx = test_start_idx + test_size
512
  if test_end_idx > len(unique_times):
513
  break
514
- train_end_idx = test_start_idx - 8
515
  if train_end_idx <= 0:
516
  continue
517
 
@@ -606,6 +660,7 @@ class DataPipeline:
606
  self.last_update = None
607
  self.status_message = "Veri yüklenmedi. 'Verileri Yenile' butonuna basın."
608
  self._backtest_cache = None
 
609
 
610
  if _HAS_JOBLIB and os.path.exists(MODEL_PATH):
611
  try:
@@ -644,21 +699,61 @@ class DataPipeline:
644
 
645
  self.featured_df = build_features(self.history_df)
646
  self._backtest_cache = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
647
 
648
  def train_model(self):
649
  if self.featured_df.empty:
650
  self.status_message = "Önce veri çekmelisiniz."
651
  return False
652
 
653
- X, y, _ = prepare_model_data(self.featured_df)
654
- if len(X) < MIN_TRAIN_SAMPLES:
655
- self.status_message = f"Yetersiz örnek: {len(X)} (min {MIN_TRAIN_SAMPLES})"
 
 
 
 
 
 
 
656
  return False
657
 
658
- # Hedef dağılımı kontrolü
659
- y_std = y.std()
660
- if y_std < 1e-3:
661
- self.status_message = f"Hedef varyansı çok düşük (std={y_std:.4f}). Model öğrenemez."
 
 
 
 
 
 
 
662
  return False
663
 
664
  self.model = train_global_model(X, y)
@@ -666,13 +761,12 @@ class DataPipeline:
666
  self.status_message = "Model eğitimi başarısız."
667
  return False
668
 
669
- # Tahmin çeşitliliği kontrolü (eğitim setinde)
670
  sample_preds = self.model.predict(X.head(500))
671
- pred_std = np.std(sample_preds)
672
- if pred_std < 1.0: # bp cinsinden
673
- self.status_message = f"⚠️ Model öğrenmedi (std={pred_std:.4f} bp). Daha fazla veri gerekli."
674
  else:
675
- self.status_message = f"Model eğitildi ({len(X)} örnek, pred_std={pred_std:.1f} bp)."
676
 
677
  if _HAS_JOBLIB:
678
  try:
@@ -733,6 +827,7 @@ pipeline = DataPipeline()
733
 
734
 
735
  def get_dashboard_metrics():
 
736
  if pipeline.featured_df.empty:
737
  return pd.DataFrame({
738
  "Metrik": ["Durum", "Veri Kaynağı", "Model"],
@@ -742,15 +837,18 @@ def get_dashboard_metrics():
742
  df = pipeline.featured_df
743
  metrics = {
744
  "Veri Kaynağı": "Public AODP API (europe)",
745
- "Time-scale": f"{HISTORY_TIME_SCALE}h",
746
  "Toplam Gözlem": len(df),
747
  "Ürün Sayısı": df["item_id"].nunique(),
748
  "Şehir Sayısı": df["city"].nunique(),
 
 
 
 
749
  "Tarih Aralığı": f"{df['timestamp'].min().date()} - {df['timestamp'].max().date()}",
750
  "Ortalama Fiyat": f"{df['avg_price'].mean():.0f} silver",
751
  "Model Durumu": "Eğitildi" if pipeline.model else "Eğitilmedi",
752
  "Tahmin Sınırı": f"±%{MAX_LOG_RETURN * 100:.0f}",
753
- "Hedef Ölçeği": f"baz puan (×{TARGET_SCALE:.0f})",
754
  "Son Güncelleme": pipeline.last_update.strftime("%Y-%m-%d %H:%M") if pipeline.last_update else "—",
755
  }
756
  return pd.DataFrame(list(metrics.items()), columns=["Metrik", "Değer"])
@@ -866,7 +964,9 @@ def get_model_performance_plot():
866
  def get_backtest_outputs():
867
  bt = pipeline.get_backtest()
868
  if "error" in bt:
869
- return pd.DataFrame({"Bilgi": [bt["error"]]}), go.Figure().update_layout(title="Backtest hatası", template="plotly_dark"), pd.DataFrame()
 
 
870
 
871
  results_df = pd.DataFrame(bt["results"])
872
  fig = go.Figure()
@@ -961,7 +1061,7 @@ with gr.Blocks(title="GOSHAWK ALBION MARKET AI", theme=gr.themes.Soft(primary_hu
961
  **Profesyonel Market Intelligence Dashboard** | Avrupa Sunucusu
962
  *LightGBM tabanlı 24 saatlik fiyat tahmini ve arbitraj analizi.*
963
 
964
- **Tahmin sınırı:** ±%15 · **Hedef ölçeği:** baz puan (×10000)
965
 
966
  **⚠️ Risk Uyarısı:** Bu uygulama yatırım tavsiyesi değildir. Model tahminleri belirsizlik içerir ve kesin kazanç garantisi sunmaz.
967
  """
@@ -973,17 +1073,21 @@ with gr.Blocks(title="GOSHAWK ALBION MARKET AI", theme=gr.themes.Soft(primary_hu
973
 
974
  with gr.Tabs():
975
  with gr.TabItem("📊 Dashboard"):
976
- dashboard_table = gr.Dataframe(label="Özet", interactive=False, wrap=True)
977
  gr.Markdown(
978
  "### Kalibrasyon\n"
979
- "- Hedef baz puan ölçeğinde (×10000) → model daha stabil öğrenir\n"
980
- "- Fiyat feature'ları `log1p` ile normalize edilir\n"
981
- "- Tahmin ±%15 aralığına clip'lenir\n"
982
- "- |tahmin| < 1% ise **belirsiz** olarak işaretlenir\n\n"
 
 
 
 
 
 
983
  "### Riskler\n"
984
- "- Model tahminleri **olasılıksaldır**\n"
985
- "- Geçmiş performans gelecek garantisi değildir\n"
986
- "- Spread ve likidite düşükse gerçekleşme fiyatı sapabilir"
987
  )
988
 
989
  with gr.TabItem("📦 100 Ürün Analizi"):
@@ -1005,7 +1109,11 @@ with gr.Blocks(title="GOSHAWK ALBION MARKET AI", theme=gr.themes.Soft(primary_hu
1005
  simulation_table = gr.Dataframe(label="Simülasyon", interactive=False)
1006
 
1007
  with gr.TabItem("🗄️ Veri Yönetimi"):
1008
- gr.Markdown("**API:** `europe.albion-online-data.com/api/v2`\n**Model:** `gos_hawk_data/models/gos_hawk_model.pkl`")
 
 
 
 
1009
  csv_files = gr.Dataframe(label="CSV Dosyaları", interactive=False,
1010
  headers=["Dosya", "Boyut (KB)", "Son Değişiklik"])
1011
 
 
95
  MODEL_PATH = os.path.join(MODEL_DIR, "gos_hawk_model.pkl")
96
 
97
  MIN_TRAIN_SAMPLES = 150
98
+ REQUEST_TIMEOUT = 60 # daha uzun timeout (saatlik veri için)
99
  MAX_RETRIES = 3
100
  RETRY_BACKOFF = 2
 
101
 
102
+ # Saatlik veri (24 saatlik hedef için daha fazla nokta → güvenilir model)
103
+ HISTORY_TIME_SCALE = 1
104
+
105
+ # Hedef ölçeği: log getiri * 10000 = baz puan
106
  TARGET_SCALE = 10000.0
107
 
108
  # Tahmin clip (log getiri, ±%15)
 
112
  # Belirsizlik eşiği (log getiri)
113
  LOW_CONFIDENCE_THRESHOLD = 0.01
114
 
115
+ # Zaman-tabanlı hedef toleransı (saat)
116
+ TARGET_TOLERANCE_HOURS = 12
117
+
118
+ # Hedef feature kolonları
119
+ FEATURE_COLS = [
120
+ "hour", "dayofweek", "is_weekend",
121
+ "lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
122
+ "chg_1h", "chg_6h", "chg_12h", "chg_24h",
123
+ "ma_6h", "ma_12h", "ma_24h", "ma_48h",
124
+ "rsi_14", "volatility_24h",
125
+ "volume_ma_24h", "volume_ratio",
126
+ "city_price_deviation",
127
+ "item_code", "city_code",
128
+ ]
129
+
130
+ PRICE_LIKE_COLS = [
131
+ "lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
132
+ "ma_6h", "ma_12h", "ma_24h", "ma_48h",
133
+ ]
134
+
135
  # ---------------------------------------------------------------------------
136
  # PUBLIC API ÇAĞRILARI
137
  # ---------------------------------------------------------------------------
 
139
 
140
  def safe_request(url: str, params: Optional[dict] = None) -> Optional[list]:
141
  headers = {
142
+ "User-Agent": "GOSHAWK-ALBION-MARKET-AI/1.4 (public-api-client)",
143
  "Accept": "application/json",
144
  }
145
  for attempt in range(MAX_RETRIES):
 
189
  return df
190
 
191
 
192
+ def fetch_history(item_ids: List[str], cities: List[str], time_scale: int = 1, qualities: str = "1") -> pd.DataFrame:
193
  all_records = []
194
+ chunk_size = 20 # saatlik veri daha ağır → daha küçük chunk
195
  for i in range(0, len(item_ids), chunk_size):
196
  chunk = item_ids[i:i+chunk_size]
197
  url = f"{API_BASE}/stats/history/{','.join(chunk)}.json"
 
209
  "avg_price": point.get("avg_price"),
210
  "item_count": point.get("item_count", 0),
211
  })
212
+ time.sleep(0.5)
213
  if not all_records:
214
  return pd.DataFrame()
215
 
 
236
  df["avg_price"] = pd.to_numeric(df["avg_price"], errors="coerce").astype("float64")
237
  df["item_count"] = pd.to_numeric(df["item_count"], errors="coerce").fillna(0).astype("float64")
238
  df = df[df["avg_price"] > 0].copy()
239
+ df = df.dropna(subset=["timestamp"]).copy()
240
 
241
  for item in df["item_id"].unique():
242
  mask = df["item_id"] == item
 
276
 
277
 
278
  def build_features(df: pd.DataFrame) -> pd.DataFrame:
279
+ """
280
+ Saatlik veride feature'lar:
281
+ - lag_1h..lag_48h: saatlik fiyat gecikmeleri
282
+ - chg_1h..chg_24h: log değişimleri
283
+ - ma_6h..ma_48h: hareketli ortalamalar
284
+ - rsi_14, volatility_24h
285
+ - volume_ratio, city_price_deviation
286
+ - target_return_24h: zaman-tabanlı (tolerance ile) 24 saat ileri log getiri × 10000
287
+ """
288
  if df.empty:
289
  return df
290
 
 
298
 
299
  group_cols = ["item_id", "city"]
300
 
301
+ # Saatlik lag'ler
302
  for lag in [1, 3, 6, 12, 24, 48]:
303
  df[f"lag_{lag}h"] = df.groupby(group_cols)["avg_price"].shift(lag)
304
 
305
  df["log_price"] = np.log(df["avg_price"].replace(0, np.nan))
306
+
307
  for chg in [1, 6, 12, 24]:
308
  df[f"chg_{chg}h"] = df.groupby(group_cols)["log_price"].shift(chg)
309
 
 
314
  )
315
 
316
  df["rsi_14"] = df.groupby(group_cols)["avg_price"].transform(lambda x: compute_rsi(x, 14))
 
317
  df["volatility_24h"] = df.groupby(group_cols)["log_price"].transform(
318
  lambda x: x.rolling(24, min_periods=6).std()
319
  )
 
320
  df["volume_ma_24h"] = df.groupby(group_cols)["item_count"].transform(
321
  lambda x: x.rolling(24, min_periods=6).mean()
322
  )
 
325
  city_avg = df.groupby(["item_id", "timestamp"])["avg_price"].transform("mean")
326
  df["city_price_deviation"] = (df["avg_price"] - city_avg) / city_avg.replace(0, np.nan)
327
 
328
+ # ZAMAN-TABANLI HEDEF: her (item_id, city) grubu için 24 saat ilerideki log getiri
329
+ df = _compute_forward_target(df)
330
+
331
+ return df
332
+
333
+
334
+ def _compute_forward_target(df: pd.DataFrame) -> pd.DataFrame:
335
+ """
336
+ Her (item_id, city) grubu için 24 saat ilerideki log fiyatını bulur.
337
+ shift(-N) yerine merge_asof ile zaman-tabanlı eşleştirme yapar (düzensiz aralıklara dayanıklı).
338
+ """
339
+ if df.empty or "log_price" not in df.columns:
340
+ df["target_return_24h"] = np.nan
341
+ return df
342
+
343
+ df["target_return_24h"] = np.nan
344
+
345
+ for (item_id, city), g in df.groupby(["item_id", "city"]):
346
+ if len(g) < 5:
347
+ continue
348
+
349
+ g = g.sort_values("timestamp")
350
+ left = pd.DataFrame({
351
+ "timestamp": g["timestamp"].values,
352
+ "log_price_now": g["log_price"].values,
353
+ "idx": g.index.values,
354
+ })
355
+ right = pd.DataFrame({
356
+ "timestamp": g["timestamp"].values,
357
+ "log_price_future": g["log_price"].values,
358
+ }).sort_values("timestamp")
359
+
360
+ left["target_timestamp"] = left["timestamp"] + pd.Timedelta(hours=24)
361
+
362
+ merged = pd.merge_asof(
363
+ left.sort_values("target_timestamp"),
364
+ right.sort_values("timestamp"),
365
+ left_on="target_timestamp",
366
+ right_on="timestamp",
367
+ direction="nearest",
368
+ tolerance=pd.Timedelta(hours=TARGET_TOLERANCE_HOURS),
369
+ suffixes=("", "_future"),
370
+ )
371
+
372
+ future_log = merged["log_price_future"].values
373
+ current_log = merged["log_price_now"].values
374
+ target_log = future_log - current_log
375
+ idx = merged["idx"].values
376
+
377
+ df.loc[idx, "target_return_24h"] = target_log * TARGET_SCALE
378
 
379
  return df
380
 
 
383
  # MODEL EĞİTİMİ
384
  # ---------------------------------------------------------------------------
385
 
 
 
 
 
 
 
 
 
 
 
 
386
 
387
  def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame]:
 
 
 
 
388
  if featured_df.empty:
389
  return pd.DataFrame(), pd.Series(dtype=float), pd.DataFrame()
390
 
 
392
  df["item_code"] = df["item_id"].astype("category").cat.codes
393
  df["city_code"] = df["city"].astype("category").cat.codes
394
 
395
+ df = df.dropna(subset=["target_return_24h", "lag_24h"]).copy()
396
 
397
+ # Fiyat-benzeri feature'ları log ölçeğine çevir
398
+ for col in PRICE_LIKE_COLS:
 
 
 
 
399
  if col in df.columns:
400
  df[col] = np.log1p(df[col].clip(lower=1.0))
401
 
 
407
  if "ma_24h" in df.columns:
408
  df["ma_24h"] = df["ma_24h"].fillna(df["lag_24h"])
409
 
410
+ for lag_col in ["lag_1h", "lag_3h", "lag_6h", "lag_12h"]:
411
+ if lag_col in df.columns:
412
+ df[lag_col] = df[lag_col].fillna(df["lag_24h"])
413
+
414
  for chg in ["chg_1h", "chg_6h", "chg_12h", "chg_24h"]:
415
  if chg in df.columns:
416
  df[chg] = df[chg].fillna(0.0)
 
423
 
424
  model_df = df.dropna(subset=FEATURE_COLS + ["target_return_24h"]).copy()
425
 
426
+ # Hedef clip (bp cinsinden ±1500)
427
  target_min = MIN_LOG_RETURN * TARGET_SCALE
428
  target_max = MAX_LOG_RETURN * TARGET_SCALE
429
  model_df["target_return_24h"] = model_df["target_return_24h"].clip(target_min, target_max)
 
441
  print(f"Yetersiz veri: {len(X)} < {MIN_TRAIN_SAMPLES}")
442
  return None
443
 
444
+ y_log_std = (y / TARGET_SCALE).std()
445
+ y_bp_std = y.std()
446
+ print(f"[TANI] train_global_model: n={len(X)}, y_bp_std={y_bp_std:.4f}, y_log_std={y_log_std:.8f}")
447
+
448
+ if y_log_std < 1e-6:
449
+ print(f"⚠️ Hedef log std çok düşük: {y_log_std:.8f}. Model öğrenemez.")
450
  return None
451
 
452
  split_idx = int(len(X) * 0.8)
453
  X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
454
  y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
455
 
 
456
  model = lgb.LGBMRegressor(
457
  objective="regression",
458
  metric="mae",
 
484
  callbacks=callbacks,
485
  )
486
 
 
487
  val_preds = model.predict(X_val)
488
+ pred_std = float(np.std(val_preds))
489
+ print(f"[TANI] val_pred_std={pred_std:.4f} bp")
490
 
491
  return model
492
 
493
 
494
  def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame) -> pd.DataFrame:
 
 
 
495
  if latest_features.empty or model is None:
496
  return pd.DataFrame()
497
 
 
500
  print(f"Eksik özellikler: {missing}")
501
  return pd.DataFrame()
502
 
 
503
  X_pred = latest_features.copy()
504
+
505
+ for col in PRICE_LIKE_COLS:
 
 
 
506
  if col in X_pred.columns:
507
  X_pred[col] = np.log1p(X_pred[col].clip(lower=1.0))
508
 
 
520
  if X_ready.empty:
521
  return pd.DataFrame()
522
 
523
+ raw_preds_bp = model.predict(X_ready)
524
+ raw_preds_log = raw_preds_bp / TARGET_SCALE
525
 
526
+ pred_std = float(np.std(raw_preds_log))
 
527
  if pred_std < 1e-5:
528
  print(f"⚠️ UYARI: Tüm tahminler aynı (std={pred_std:.8f}).")
529
 
 
530
  clipped_log = np.clip(raw_preds_log, MIN_LOG_RETURN, MAX_LOG_RETURN)
 
531
  pred_return_pct = (np.exp(clipped_log) - 1) * 100
532
 
533
  result = X_pred.loc[X_ready.index, ["item_id", "city", "avg_price", "item_count",
 
553
 
554
  df = featured_df.sort_values("timestamp").copy()
555
  unique_times = df["timestamp"].sort_values().unique()
556
+ if len(unique_times) < 100:
557
  return {"error": "Yeterli zaman noktası yok."}
558
 
559
  test_size = len(unique_times) // (n_splits + 1)
 
561
  all_predictions = []
562
 
563
  for i in range(n_splits):
564
+ test_start_idx = test_size * (i + 1) + 24
565
  test_end_idx = test_start_idx + test_size
566
  if test_end_idx > len(unique_times):
567
  break
568
+ train_end_idx = test_start_idx - 48
569
  if train_end_idx <= 0:
570
  continue
571
 
 
660
  self.last_update = None
661
  self.status_message = "Veri yüklenmedi. 'Verileri Yenile' butonuna basın."
662
  self._backtest_cache = None
663
+ self._diagnostics = {}
664
 
665
  if _HAS_JOBLIB and os.path.exists(MODEL_PATH):
666
  try:
 
699
 
700
  self.featured_df = build_features(self.history_df)
701
  self._backtest_cache = None
702
+ self._update_diagnostics()
703
+
704
+ def _update_diagnostics(self):
705
+ df = self.featured_df
706
+ if df.empty:
707
+ self._diagnostics = {}
708
+ return
709
+ n_total = len(df)
710
+ n_with_target = int(df["target_return_24h"].notna().sum())
711
+ n_groups = int(df.groupby(["item_id", "city"]).ngroups)
712
+ if n_with_target > 0:
713
+ target_bp = df["target_return_24h"].dropna()
714
+ target_std_bp = float(target_bp.std())
715
+ target_std_log = target_std_bp / TARGET_SCALE
716
+ else:
717
+ target_std_bp = 0.0
718
+ target_std_log = 0.0
719
+ self._diagnostics = {
720
+ "n_total": n_total,
721
+ "n_with_target": n_with_target,
722
+ "n_groups": n_groups,
723
+ "target_std_bp": target_std_bp,
724
+ "target_std_log": target_std_log,
725
+ }
726
+ print(f"[TANI] total={n_total}, hedefli={n_with_target}, grup={n_groups}, "
727
+ f"std_bp={target_std_bp:.4f}, std_log={target_std_log:.8f}")
728
 
729
  def train_model(self):
730
  if self.featured_df.empty:
731
  self.status_message = "Önce veri çekmelisiniz."
732
  return False
733
 
734
+ diag = self._diagnostics or {}
735
+ n_with_target = diag.get("n_with_target", 0)
736
+ std_log = diag.get("target_std_log", 0.0)
737
+
738
+ if n_with_target < MIN_TRAIN_SAMPLES:
739
+ self.status_message = (
740
+ f"Yetersiz hedefli örnek: {n_with_target} < {MIN_TRAIN_SAMPLES}. "
741
+ f"Toplam: {diag.get('n_total', 0)}, grup: {diag.get('n_groups', 0)}. "
742
+ f"24 saatlik ileri veri bulunamadı — API çok seyrek olabilir."
743
+ )
744
  return False
745
 
746
+ if std_log < 1e-6:
747
+ self.status_message = (
748
+ f"Hedef varyansı çok düşük (std_log={std_log:.8f}, "
749
+ f"std_bp={diag.get('target_std_bp', 0):.4f}). Model öğrenemez. "
750
+ f"API'den daha fazla veri çekmeyi deneyin."
751
+ )
752
+ return False
753
+
754
+ X, y, _ = prepare_model_data(self.featured_df)
755
+ if len(X) < MIN_TRAIN_SAMPLES:
756
+ self.status_message = f"Model eğitimi için yetersiz örnek: {len(X)} < {MIN_TRAIN_SAMPLES}"
757
  return False
758
 
759
  self.model = train_global_model(X, y)
 
761
  self.status_message = "Model eğitimi başarısız."
762
  return False
763
 
 
764
  sample_preds = self.model.predict(X.head(500))
765
+ pred_std = float(np.std(sample_preds))
766
+ if pred_std < 1.0:
767
+ self.status_message = f"⚠️ Model zayıf öğrendi (pred_std={pred_std:.2f} bp). Daha fazla veri gerekli."
768
  else:
769
+ self.status_message = f"✓ Model eğitildi ({len(X)} örnek, pred_std={pred_std:.1f} bp)."
770
 
771
  if _HAS_JOBLIB:
772
  try:
 
827
 
828
 
829
  def get_dashboard_metrics():
830
+ diag = pipeline._diagnostics
831
  if pipeline.featured_df.empty:
832
  return pd.DataFrame({
833
  "Metrik": ["Durum", "Veri Kaynağı", "Model"],
 
837
  df = pipeline.featured_df
838
  metrics = {
839
  "Veri Kaynağı": "Public AODP API (europe)",
840
+ "Time-scale": f"{HISTORY_TIME_SCALE}h (saatlik)",
841
  "Toplam Gözlem": len(df),
842
  "Ürün Sayısı": df["item_id"].nunique(),
843
  "Şehir Sayısı": df["city"].nunique(),
844
+ "Grup Sayısı": diag.get("n_groups", 0),
845
+ "Hedefli Satır": diag.get("n_with_target", 0),
846
+ "Hedef std (log)": f"{diag.get('target_std_log', 0):.6f}",
847
+ "Hedef std (bp)": f"{diag.get('target_std_bp', 0):.2f}",
848
  "Tarih Aralığı": f"{df['timestamp'].min().date()} - {df['timestamp'].max().date()}",
849
  "Ortalama Fiyat": f"{df['avg_price'].mean():.0f} silver",
850
  "Model Durumu": "Eğitildi" if pipeline.model else "Eğitilmedi",
851
  "Tahmin Sınırı": f"±%{MAX_LOG_RETURN * 100:.0f}",
 
852
  "Son Güncelleme": pipeline.last_update.strftime("%Y-%m-%d %H:%M") if pipeline.last_update else "—",
853
  }
854
  return pd.DataFrame(list(metrics.items()), columns=["Metrik", "Değer"])
 
964
  def get_backtest_outputs():
965
  bt = pipeline.get_backtest()
966
  if "error" in bt:
967
+ return (pd.DataFrame({"Bilgi": [bt["error"]]}),
968
+ go.Figure().update_layout(title="Backtest hatası", template="plotly_dark"),
969
+ pd.DataFrame())
970
 
971
  results_df = pd.DataFrame(bt["results"])
972
  fig = go.Figure()
 
1061
  **Profesyonel Market Intelligence Dashboard** | Avrupa Sunucusu
1062
  *LightGBM tabanlı 24 saatlik fiyat tahmini ve arbitraj analizi.*
1063
 
1064
+ **Tahmin sınırı:** ±%15 · **Hedef ölçeği:** baz puan · **Veri:** saatlik · **Hedef:** zaman-tabanlı (24h, tolerans ±12h)
1065
 
1066
  **⚠️ Risk Uyarısı:** Bu uygulama yatırım tavsiyesi değildir. Model tahminleri belirsizlik içerir ve kesin kazanç garantisi sunmaz.
1067
  """
 
1073
 
1074
  with gr.Tabs():
1075
  with gr.TabItem("📊 Dashboard"):
1076
+ dashboard_table = gr.Dataframe(label="Özet & Teşhis", interactive=False, wrap=True)
1077
  gr.Markdown(
1078
  "### Kalibrasyon\n"
1079
+ "- Saatlik veri (`time-scale=1`) → 24h hedef için ~24 nokta\n"
1080
+ "- Hedef **zaman-tabanlı** (merge_asof, tolerans ±12h) — düzensiz aralıklara dayanıklı\n"
1081
+ "- Hedef baz puan (×10000) → model daha stabil öğrenir\n"
1082
+ "- Fiyat feature'ları `log1p` ile normalize\n"
1083
+ "- Tahmin ±%15 aralığına clip\n"
1084
+ "- |tahmin| < 1% ise **belirsiz** işaretlenir\n\n"
1085
+ "### Teşhis sütunları\n"
1086
+ "- **Hedefli Satır:** `target_return_24h` NaN olmayan satır sayısı\n"
1087
+ "- **Hedef std (log):** 0.005-0.05 arası ise model öğrenebilir\n"
1088
+ "- **Model Durumu:** Eğitildi = hazır\n\n"
1089
  "### Riskler\n"
1090
+ "- Tahminler olasılıksaldır; kesin kazanç garantisi yoktur"
 
 
1091
  )
1092
 
1093
  with gr.TabItem("📦 100 Ürün Analizi"):
 
1109
  simulation_table = gr.Dataframe(label="Simülasyon", interactive=False)
1110
 
1111
  with gr.TabItem("🗄️ Veri Yönetimi"):
1112
+ gr.Markdown(
1113
+ "**API:** `europe.albion-online-data.com/api/v2`\n"
1114
+ "**Time-scale:** 1h (saatlik)\n"
1115
+ "**Model:** `gos_hawk_data/models/gos_hawk_model.pkl`"
1116
+ )
1117
  csv_files = gr.Dataframe(label="CSV Dosyaları", interactive=False,
1118
  headers=["Dosya", "Boyut (KB)", "Son Değişiklik"])
1119