Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -35,12 +35,8 @@ warnings.filterwarnings("ignore")
|
|
| 35 |
API_BASE = "https://europe.albion-online-data.com/api/v2"
|
| 36 |
|
| 37 |
CITIES = [
|
| 38 |
-
"Bridgewatch",
|
| 39 |
-
"
|
| 40 |
-
"Lymhurst",
|
| 41 |
-
"Martlock",
|
| 42 |
-
"Thetford",
|
| 43 |
-
"Caerleon",
|
| 44 |
]
|
| 45 |
|
| 46 |
_ITEM_POOL = [
|
|
@@ -94,35 +90,26 @@ TRANSACTION_TAX_RATE = 0.08
|
|
| 94 |
|
| 95 |
DATA_DIR = "gos_hawk_data"
|
| 96 |
os.makedirs(DATA_DIR, exist_ok=True)
|
| 97 |
-
|
| 98 |
MODEL_DIR = os.path.join(DATA_DIR, "models")
|
| 99 |
os.makedirs(MODEL_DIR, exist_ok=True)
|
| 100 |
MODEL_PATH = os.path.join(MODEL_DIR, "gos_hawk_model.pkl")
|
| 101 |
|
| 102 |
-
PURGE_HOURS = 24
|
| 103 |
-
EMBARGO_HOURS = 24
|
| 104 |
MIN_TRAIN_SAMPLES = 150
|
| 105 |
-
|
| 106 |
REQUEST_TIMEOUT = 30
|
| 107 |
MAX_RETRIES = 3
|
| 108 |
RETRY_BACKOFF = 2
|
| 109 |
-
|
| 110 |
HISTORY_TIME_SCALE = 6
|
| 111 |
|
| 112 |
-
#
|
| 113 |
-
#
|
| 114 |
-
|
| 115 |
|
| 116 |
-
#
|
| 117 |
MAX_LOG_RETURN = 0.15
|
| 118 |
MIN_LOG_RETURN = -0.15
|
| 119 |
|
| 120 |
-
#
|
| 121 |
-
|
| 122 |
-
TARGET_CLIP_UPPER_Q = 0.98
|
| 123 |
-
|
| 124 |
-
# Güven eşiği: bu log getirinin altındaki tahminler "belirsiz" sayılır
|
| 125 |
-
LOW_CONFIDENCE_THRESHOLD = 0.02
|
| 126 |
|
| 127 |
# ---------------------------------------------------------------------------
|
| 128 |
# PUBLIC API ÇAĞRILARI
|
|
@@ -131,7 +118,7 @@ LOW_CONFIDENCE_THRESHOLD = 0.02
|
|
| 131 |
|
| 132 |
def safe_request(url: str, params: Optional[dict] = None) -> Optional[list]:
|
| 133 |
headers = {
|
| 134 |
-
"User-Agent": "GOSHAWK-ALBION-MARKET-AI/1.
|
| 135 |
"Accept": "application/json",
|
| 136 |
}
|
| 137 |
for attempt in range(MAX_RETRIES):
|
|
@@ -145,7 +132,6 @@ def safe_request(url: str, params: Optional[dict] = None) -> Optional[list]:
|
|
| 145 |
time.sleep(wait)
|
| 146 |
continue
|
| 147 |
elif resp.status_code >= 500:
|
| 148 |
-
print(f"Sunucu hatası {resp.status_code}. Yeniden deneniyor...")
|
| 149 |
time.sleep(RETRY_BACKOFF * (attempt + 1))
|
| 150 |
continue
|
| 151 |
else:
|
|
@@ -166,10 +152,8 @@ def fetch_current_prices(item_ids: List[str], cities: List[str], qualities: str
|
|
| 166 |
chunk_size = 50
|
| 167 |
for i in range(0, len(item_ids), chunk_size):
|
| 168 |
chunk = item_ids[i:i+chunk_size]
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
url = f"{API_BASE}/stats/prices/{items_str}.json"
|
| 172 |
-
params = {"locations": locations_str, "qualities": qualities}
|
| 173 |
data = safe_request(url, params)
|
| 174 |
if data:
|
| 175 |
all_data.extend(data)
|
|
@@ -181,9 +165,6 @@ def fetch_current_prices(item_ids: List[str], cities: List[str], qualities: str
|
|
| 181 |
for col in ["sell_price_min", "sell_price_max", "buy_price_min", "buy_price_max"]:
|
| 182 |
if col in df.columns:
|
| 183 |
df[col] = pd.to_numeric(df[col], errors="coerce").astype("float64")
|
| 184 |
-
for col in ["sell_price_min_date", "sell_price_max_date", "buy_price_min_date", "buy_price_max_date"]:
|
| 185 |
-
if col in df.columns:
|
| 186 |
-
df[col] = pd.to_datetime(df[col], errors="coerce")
|
| 187 |
return df
|
| 188 |
|
| 189 |
|
|
@@ -192,21 +173,17 @@ def fetch_history(item_ids: List[str], cities: List[str], time_scale: int = 6, q
|
|
| 192 |
chunk_size = 30
|
| 193 |
for i in range(0, len(item_ids), chunk_size):
|
| 194 |
chunk = item_ids[i:i+chunk_size]
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
url = f"{API_BASE}/stats/history/{items_str}.json"
|
| 198 |
-
params = {"locations": locations_str, "qualities": qualities, "time-scale": time_scale}
|
| 199 |
data = safe_request(url, params)
|
| 200 |
if data:
|
| 201 |
for entry in data:
|
| 202 |
item_id = entry.get("item_id")
|
| 203 |
loc = entry.get("location")
|
| 204 |
-
quality = entry.get("quality", 1)
|
| 205 |
for point in entry.get("data", []):
|
| 206 |
all_records.append({
|
| 207 |
"item_id": item_id,
|
| 208 |
"city": loc,
|
| 209 |
-
"quality": quality,
|
| 210 |
"timestamp": point.get("timestamp"),
|
| 211 |
"avg_price": point.get("avg_price"),
|
| 212 |
"item_count": point.get("item_count", 0),
|
|
@@ -225,13 +202,10 @@ def fetch_history(item_ids: List[str], cities: List[str], time_scale: int = 6, q
|
|
| 225 |
def check_api_health() -> Tuple[bool, str]:
|
| 226 |
try:
|
| 227 |
url = f"{API_BASE}/stats/prices/T4_BAG.json"
|
| 228 |
-
|
| 229 |
-
data
|
| 230 |
-
if data is not None:
|
| 231 |
-
return True, "Public API erişilebilir."
|
| 232 |
-
return False, "Public API yanıt vermedi."
|
| 233 |
except Exception as e:
|
| 234 |
-
return False,
|
| 235 |
|
| 236 |
|
| 237 |
def clean_market_data(df: pd.DataFrame) -> pd.DataFrame:
|
|
@@ -239,22 +213,19 @@ def clean_market_data(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 239 |
return df
|
| 240 |
df = df.copy()
|
| 241 |
df["avg_price"] = pd.to_numeric(df["avg_price"], errors="coerce").astype("float64")
|
| 242 |
-
|
| 243 |
-
df["item_count"] = pd.to_numeric(df["item_count"], errors="coerce").fillna(0).astype("float64")
|
| 244 |
df = df[df["avg_price"] > 0].copy()
|
| 245 |
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
df.loc[mask, "avg_price"] = df.loc[mask, "avg_price"].clip(lower, upper)
|
| 255 |
|
| 256 |
-
|
| 257 |
-
return df
|
| 258 |
|
| 259 |
|
| 260 |
def save_to_csv(df: pd.DataFrame, name: str) -> str:
|
|
@@ -279,8 +250,7 @@ def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series:
|
|
| 279 |
gain = delta.clip(lower=0).rolling(period).mean()
|
| 280 |
loss = (-delta.clip(upper=0)).rolling(period).mean()
|
| 281 |
rs = gain / loss.replace(0, np.nan)
|
| 282 |
-
|
| 283 |
-
return rsi.fillna(50)
|
| 284 |
|
| 285 |
|
| 286 |
def build_features(df: pd.DataFrame) -> pd.DataFrame:
|
|
@@ -324,8 +294,8 @@ def build_features(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 324 |
city_avg = df.groupby(["item_id", "timestamp"])["avg_price"].transform("mean")
|
| 325 |
df["city_price_deviation"] = (df["avg_price"] - city_avg) / city_avg.replace(0, np.nan)
|
| 326 |
|
| 327 |
-
#
|
| 328 |
-
df["target_return_24h"] = df.groupby(group_cols)["log_price"].shift(-4)
|
| 329 |
|
| 330 |
return df
|
| 331 |
|
|
@@ -347,6 +317,10 @@ FEATURE_COLS = [
|
|
| 347 |
|
| 348 |
|
| 349 |
def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
if featured_df.empty:
|
| 351 |
return pd.DataFrame(), pd.Series(dtype=float), pd.DataFrame()
|
| 352 |
|
|
@@ -356,22 +330,22 @@ def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Seri
|
|
| 356 |
|
| 357 |
df = df.dropna(subset=["target_return_24h", "lag_1h", "lag_24h"]).copy()
|
| 358 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
if "lag_48h" in df.columns:
|
| 360 |
df["lag_48h"] = df["lag_48h"].fillna(df["lag_24h"])
|
| 361 |
if "ma_48h" in df.columns:
|
| 362 |
df["ma_48h"] = df["ma_48h"].fillna(df["ma_24h"])
|
| 363 |
if "ma_24h" in df.columns:
|
| 364 |
df["ma_24h"] = df["ma_24h"].fillna(df["lag_24h"])
|
| 365 |
-
if "ma_12h" in df.columns:
|
| 366 |
-
df["ma_12h"] = df["ma_12h"].fillna(df.get("lag_12h", df["lag_24h"]))
|
| 367 |
-
if "ma_6h" in df.columns:
|
| 368 |
-
df["ma_6h"] = df["ma_6h"].fillna(df.get("lag_6h", df["lag_24h"]))
|
| 369 |
-
if "lag_3h" in df.columns:
|
| 370 |
-
df["lag_3h"] = df["lag_3h"].fillna(df.get("lag_6h", df["lag_24h"]))
|
| 371 |
-
if "lag_6h" in df.columns:
|
| 372 |
-
df["lag_6h"] = df["lag_6h"].fillna(df.get("lag_12h", df["lag_24h"]))
|
| 373 |
-
if "lag_12h" in df.columns:
|
| 374 |
-
df["lag_12h"] = df["lag_12h"].fillna(df["lag_24h"])
|
| 375 |
|
| 376 |
for chg in ["chg_1h", "chg_6h", "chg_12h", "chg_24h"]:
|
| 377 |
if chg in df.columns:
|
|
@@ -385,17 +359,15 @@ def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Seri
|
|
| 385 |
|
| 386 |
model_df = df.dropna(subset=FEATURE_COLS + ["target_return_24h"]).copy()
|
| 387 |
|
| 388 |
-
# KRİTİK
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
model_df["target_return_24h"] = model_df["target_return_24h"].clip(
|
| 393 |
-
max(lower, MIN_LOG_RETURN), min(upper, MAX_LOG_RETURN)
|
| 394 |
-
)
|
| 395 |
|
| 396 |
X = model_df[FEATURE_COLS].astype("float64")
|
| 397 |
y = model_df["target_return_24h"].astype("float64")
|
| 398 |
-
meta = model_df[["item_id", "city", "timestamp", "avg_price"
|
|
|
|
| 399 |
|
| 400 |
return X, y, meta
|
| 401 |
|
|
@@ -405,32 +377,38 @@ def train_global_model(X: pd.DataFrame, y: pd.Series) -> Optional[lgb.LGBMRegres
|
|
| 405 |
print(f"Yetersiz veri: {len(X)} < {MIN_TRAIN_SAMPLES}")
|
| 406 |
return None
|
| 407 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 408 |
split_idx = int(len(X) * 0.8)
|
| 409 |
X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
|
| 410 |
y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
|
| 411 |
|
| 412 |
-
#
|
| 413 |
model = lgb.LGBMRegressor(
|
| 414 |
objective="regression",
|
| 415 |
metric="mae",
|
| 416 |
boosting_type="gbdt",
|
| 417 |
-
num_leaves=
|
| 418 |
-
max_depth=
|
| 419 |
-
min_child_samples=
|
| 420 |
-
learning_rate=0.
|
| 421 |
-
n_estimators=
|
| 422 |
-
feature_fraction=0.
|
| 423 |
-
bagging_fraction=0.
|
| 424 |
bagging_freq=5,
|
| 425 |
-
reg_alpha=0.
|
| 426 |
-
reg_lambda=
|
| 427 |
verbose=-1,
|
| 428 |
random_state=42,
|
| 429 |
n_jobs=-1,
|
| 430 |
)
|
| 431 |
|
| 432 |
callbacks = [
|
| 433 |
-
lgb.early_stopping(stopping_rounds=
|
| 434 |
lgb.log_evaluation(period=0),
|
| 435 |
]
|
| 436 |
|
|
@@ -441,15 +419,17 @@ def train_global_model(X: pd.DataFrame, y: pd.Series) -> Optional[lgb.LGBMRegres
|
|
| 441 |
callbacks=callbacks,
|
| 442 |
)
|
| 443 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
return model
|
| 445 |
|
| 446 |
|
| 447 |
def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame) -> pd.DataFrame:
|
| 448 |
"""
|
| 449 |
-
|
| 450 |
-
1. Ham tahmin clip'lenir (aşırı değerleri engelle).
|
| 451 |
-
2. Tahminin mutlak değeri çok küçükse "belirsiz" işaretlenir.
|
| 452 |
-
3. exp() güvenli aralıkta uygulanır.
|
| 453 |
"""
|
| 454 |
if latest_features.empty or model is None:
|
| 455 |
return pd.DataFrame()
|
|
@@ -459,27 +439,51 @@ def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame)
|
|
| 459 |
print(f"Eksik özellikler: {missing}")
|
| 460 |
return pd.DataFrame()
|
| 461 |
|
| 462 |
-
|
| 463 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 464 |
return pd.DataFrame()
|
| 465 |
|
| 466 |
-
|
|
|
|
| 467 |
|
| 468 |
-
#
|
| 469 |
-
|
|
|
|
|
|
|
| 470 |
|
| 471 |
-
#
|
| 472 |
-
|
| 473 |
|
| 474 |
-
|
| 475 |
-
result["raw_log_pred"] = raw_preds
|
| 476 |
-
result["clipped_log_pred"] = clipped_preds
|
| 477 |
-
result["predicted_log_return_24h"] = clipped_preds
|
| 478 |
-
result["predicted_return_pct"] = pred_return_pct
|
| 479 |
-
result["predicted_price_24h"] = result["avg_price"] * np.exp(clipped_preds)
|
| 480 |
|
| 481 |
-
|
| 482 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 483 |
|
| 484 |
return result
|
| 485 |
|
|
@@ -495,9 +499,8 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 495 |
|
| 496 |
df = featured_df.sort_values("timestamp").copy()
|
| 497 |
unique_times = df["timestamp"].sort_values().unique()
|
| 498 |
-
|
| 499 |
if len(unique_times) < 50:
|
| 500 |
-
return {"error": "
|
| 501 |
|
| 502 |
test_size = len(unique_times) // (n_splits + 1)
|
| 503 |
results = []
|
|
@@ -508,7 +511,6 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 508 |
test_end_idx = test_start_idx + test_size
|
| 509 |
if test_end_idx > len(unique_times):
|
| 510 |
break
|
| 511 |
-
|
| 512 |
train_end_idx = test_start_idx - 8
|
| 513 |
if train_end_idx <= 0:
|
| 514 |
continue
|
|
@@ -529,31 +531,24 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 529 |
continue
|
| 530 |
|
| 531 |
model = lgb.LGBMRegressor(
|
| 532 |
-
objective="regression",
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
min_child_samples=50,
|
| 537 |
-
learning_rate=0.03,
|
| 538 |
-
n_estimators=300,
|
| 539 |
-
reg_alpha=0.1,
|
| 540 |
-
reg_lambda=1.0,
|
| 541 |
-
verbose=-1,
|
| 542 |
-
random_state=42,
|
| 543 |
-
n_jobs=-1,
|
| 544 |
)
|
| 545 |
model.fit(X_train, y_train)
|
| 546 |
|
| 547 |
-
|
| 548 |
-
preds = np.clip(
|
|
|
|
| 549 |
|
| 550 |
-
mae = mean_absolute_error(
|
| 551 |
-
rmse = np.sqrt(mean_squared_error(
|
| 552 |
-
direction_correct = np.mean((
|
| 553 |
|
| 554 |
-
baseline_preds = np.zeros_like(
|
| 555 |
-
baseline_mae = mean_absolute_error(
|
| 556 |
-
baseline_rmse = np.sqrt(mean_squared_error(
|
| 557 |
|
| 558 |
results.append({
|
| 559 |
"split": i + 1,
|
|
@@ -569,11 +564,11 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 569 |
|
| 570 |
test_meta = meta_test.copy()
|
| 571 |
test_meta["predicted_return"] = preds
|
| 572 |
-
test_meta["actual_return"] =
|
| 573 |
all_predictions.append(test_meta)
|
| 574 |
|
| 575 |
if not results:
|
| 576 |
-
return {"error": "Backtest tamamlanamadı
|
| 577 |
|
| 578 |
if all_predictions:
|
| 579 |
sim_df = pd.concat(all_predictions)
|
|
@@ -583,12 +578,10 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 583 |
sim_trades = sim_df[sim_df["trade_signal"]].copy()
|
| 584 |
|
| 585 |
if len(sim_trades) > 0:
|
| 586 |
-
avg_net_return = sim_trades["net_return"].mean()
|
| 587 |
-
win_rate = (sim_trades["net_return"] > 0).mean()
|
| 588 |
simulation = {
|
| 589 |
"total_trades": int(len(sim_trades)),
|
| 590 |
-
"avg_net_return_per_trade_pct": round(float(
|
| 591 |
-
"win_rate_pct": round(float(
|
| 592 |
"profitable_trades": int((sim_trades["net_return"] > 0).sum()),
|
| 593 |
}
|
| 594 |
else:
|
|
@@ -596,11 +589,7 @@ def walk_forward_backtest(featured_df: pd.DataFrame, n_splits: int = 3) -> Dict:
|
|
| 596 |
else:
|
| 597 |
simulation = {}
|
| 598 |
|
| 599 |
-
return {
|
| 600 |
-
"results": results,
|
| 601 |
-
"simulation": simulation,
|
| 602 |
-
"n_splits_completed": len(results),
|
| 603 |
-
}
|
| 604 |
|
| 605 |
|
| 606 |
# ---------------------------------------------------------------------------
|
|
@@ -617,23 +606,19 @@ class DataPipeline:
|
|
| 617 |
self.last_update = None
|
| 618 |
self.status_message = "Veri yüklenmedi. 'Verileri Yenile' butonuna basın."
|
| 619 |
self._backtest_cache = None
|
| 620 |
-
self.data_source = "PUBLIC_API_EUROPE"
|
| 621 |
|
| 622 |
if _HAS_JOBLIB and os.path.exists(MODEL_PATH):
|
| 623 |
try:
|
| 624 |
self.model = joblib.load(MODEL_PATH)
|
| 625 |
-
self.status_message = "Kayıtlı model yüklendi.
|
| 626 |
-
print("Kayıtlı model yüklendi.")
|
| 627 |
except Exception as e:
|
| 628 |
print(f"Model yükleme hatası: {e}")
|
| 629 |
|
| 630 |
-
def fetch_data(self, item_ids
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
if cities is None:
|
| 634 |
-
cities = CITIES
|
| 635 |
|
| 636 |
-
self.status_message =
|
| 637 |
ok, msg = check_api_health()
|
| 638 |
if not ok:
|
| 639 |
self.status_message = f"API erişilemiyor: {msg}"
|
|
@@ -647,15 +632,12 @@ class DataPipeline:
|
|
| 647 |
return
|
| 648 |
|
| 649 |
if hist.empty:
|
| 650 |
-
self.status_message = "Tarihsel veri alınamadı.
|
| 651 |
return
|
| 652 |
|
| 653 |
self.history_df = clean_market_data(hist)
|
| 654 |
self.prices_df = prices
|
| 655 |
-
self.status_message = (
|
| 656 |
-
f"✓ Public API'den alındı: {len(self.history_df)} satır tarihsel, "
|
| 657 |
-
f"{len(self.prices_df)} anlık fiyat. (time-scale={HISTORY_TIME_SCALE}h)"
|
| 658 |
-
)
|
| 659 |
|
| 660 |
save_to_csv(self.history_df, "history")
|
| 661 |
save_to_csv(self.prices_df, "prices")
|
|
@@ -670,11 +652,13 @@ class DataPipeline:
|
|
| 670 |
|
| 671 |
X, y, _ = prepare_model_data(self.featured_df)
|
| 672 |
if len(X) < MIN_TRAIN_SAMPLES:
|
| 673 |
-
self.status_message = (
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
|
|
|
|
|
|
| 678 |
return False
|
| 679 |
|
| 680 |
self.model = train_global_model(X, y)
|
|
@@ -682,17 +666,19 @@ class DataPipeline:
|
|
| 682 |
self.status_message = "Model eğitimi başarısız."
|
| 683 |
return False
|
| 684 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 685 |
if _HAS_JOBLIB:
|
| 686 |
try:
|
| 687 |
joblib.dump(self.model, MODEL_PATH)
|
| 688 |
-
self.status_message = (
|
| 689 |
-
f"Model eğitildi ve kaydedildi ({len(X)} örnek). "
|
| 690 |
-
f"Tahminler ±%15 aralığına sınırlandırıldı."
|
| 691 |
-
)
|
| 692 |
except Exception as e:
|
| 693 |
-
|
| 694 |
-
else:
|
| 695 |
-
self.status_message = f"Model eğitildi ({len(X)} örnek). Tahminler ±%15 aralığına sınırlandırıldı."
|
| 696 |
|
| 697 |
self.last_update = dt.datetime.now()
|
| 698 |
return True
|
|
@@ -720,12 +706,10 @@ class DataPipeline:
|
|
| 720 |
else:
|
| 721 |
preds["spread"] = np.nan
|
| 722 |
|
| 723 |
-
# Gelişmiş güvenilirlik: volatilite + RSI aşırı bölge + volume
|
| 724 |
vol_score = (1.0 - preds["volatility_24h"].clip(0, 0.2) / 0.2).fillna(0.5)
|
| 725 |
rsi_penalty = ((preds["rsi_14"] - 50).abs() / 50).clip(0, 1).fillna(0.5) * 0.2
|
| 726 |
vol_ratio_score = preds["volume_ratio"].clip(0, 2).fillna(1.0) / 2
|
| 727 |
base_conf = vol_score * 0.6 + vol_ratio_score * 0.4 - rsi_penalty
|
| 728 |
-
# Belirsiz tahminleri cezalandır
|
| 729 |
certainty_penalty = preds["is_uncertain"].astype(float) * 0.4
|
| 730 |
preds["confidence"] = (base_conf - certainty_penalty).clip(0, 1).fillna(0.3)
|
| 731 |
|
|
@@ -744,33 +728,29 @@ pipeline = DataPipeline()
|
|
| 744 |
|
| 745 |
|
| 746 |
# ---------------------------------------------------------------------------
|
| 747 |
-
# GRADIO
|
| 748 |
# ---------------------------------------------------------------------------
|
| 749 |
|
| 750 |
|
| 751 |
def get_dashboard_metrics():
|
| 752 |
if pipeline.featured_df.empty:
|
| 753 |
return pd.DataFrame({
|
| 754 |
-
"Metrik": ["Durum", "Veri Kaynağı", "Model
|
| 755 |
-
"Değer": [
|
| 756 |
-
"Veri yok - 'Verileri Yenile' butonuna tıklayın.",
|
| 757 |
-
"Public AODP API (europe)",
|
| 758 |
-
"Kayıtlı model var" if pipeline.model is not None else "Yok",
|
| 759 |
-
]
|
| 760 |
})
|
| 761 |
|
| 762 |
df = pipeline.featured_df
|
| 763 |
metrics = {
|
| 764 |
-
"Veri Kaynağı": "Public AODP API (europe
|
| 765 |
-
"Time-scale": f"{HISTORY_TIME_SCALE}
|
| 766 |
"Toplam Gözlem": len(df),
|
| 767 |
"Ürün Sayısı": df["item_id"].nunique(),
|
| 768 |
"Şehir Sayısı": df["city"].nunique(),
|
| 769 |
"Tarih Aralığı": f"{df['timestamp'].min().date()} - {df['timestamp'].max().date()}",
|
| 770 |
"Ortalama Fiyat": f"{df['avg_price'].mean():.0f} silver",
|
| 771 |
-
"Ortalama Hacim": f"{df['item_count'].mean():.1f}",
|
| 772 |
"Model Durumu": "Eğitildi" if pipeline.model else "Eğitilmedi",
|
| 773 |
"Tahmin Sınırı": f"±%{MAX_LOG_RETURN * 100:.0f}",
|
|
|
|
| 774 |
"Son Güncelleme": pipeline.last_update.strftime("%Y-%m-%d %H:%M") if pipeline.last_update else "—",
|
| 775 |
}
|
| 776 |
return pd.DataFrame(list(metrics.items()), columns=["Metrik", "Değer"])
|
|
@@ -796,22 +776,16 @@ def get_analysis_table():
|
|
| 796 |
latest["Mevcut Fiyat"] = latest["avg_price"].round(0)
|
| 797 |
latest["Hacim"] = latest["item_count"].fillna(0).astype(int)
|
| 798 |
latest["Spread"] = (latest["sell_price_min"] - latest["buy_price_max"]).round(0)
|
| 799 |
-
latest["Likidite"] = pd.cut(
|
| 800 |
-
|
| 801 |
-
bins=[-1, 10, 50, 200, np.inf],
|
| 802 |
-
labels=["Düşük", "Orta", "İyi", "Yüksek"]
|
| 803 |
-
).astype(str)
|
| 804 |
|
| 805 |
-
|
| 806 |
-
return latest[cols].sort_values("Hacim", ascending=False).head(100)
|
| 807 |
|
| 808 |
|
| 809 |
def get_prediction_table():
|
| 810 |
preds = pipeline.get_latest_predictions()
|
| 811 |
if preds.empty:
|
| 812 |
-
return pd.DataFrame({
|
| 813 |
-
"Bilgi": ["Tahmin üretilemedi. Veri çekip model eğitin."]
|
| 814 |
-
})
|
| 815 |
|
| 816 |
preds = preds.copy()
|
| 817 |
preds["Ürün"] = preds["item_id"].apply(item_display_name)
|
|
@@ -822,58 +796,45 @@ def get_prediction_table():
|
|
| 822 |
preds["Güvenilirlik"] = (preds["confidence"] * 100).round(0).astype(int).astype(str) + "%"
|
| 823 |
preds["Spread"] = preds["spread"].round(0)
|
| 824 |
preds["Net Getiri % (maliyet sonrası)"] = preds["net_return_pct"].round(2)
|
| 825 |
-
preds["Likidite"] = pd.cut(
|
| 826 |
-
|
| 827 |
-
bins=[-1, 10, 50, 200, np.inf],
|
| 828 |
-
labels=["Düşük", "Orta", "İyi", "Yüksek"]
|
| 829 |
-
).astype(str)
|
| 830 |
-
# Belirsiz tahminleri işaretle
|
| 831 |
preds["Not"] = np.where(preds["is_uncertain"], "⚠️ Belirsiz", "")
|
| 832 |
|
| 833 |
-
cols = [
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
"Net Getiri % (maliyet sonrası)", "Not"
|
| 837 |
-
]
|
| 838 |
return preds[cols].sort_values("Tahmini Değişim %", ascending=False)
|
| 839 |
|
| 840 |
|
| 841 |
def get_arbitrage_table():
|
| 842 |
if pipeline.prices_df.empty:
|
| 843 |
-
return pd.DataFrame({"Bilgi": ["Anlık fiyat
|
| 844 |
|
| 845 |
prices = pipeline.prices_df.copy()
|
| 846 |
prices["sell_price_min"] = pd.to_numeric(prices["sell_price_min"], errors="coerce").astype("float64")
|
| 847 |
prices["buy_price_max"] = pd.to_numeric(prices["buy_price_max"], errors="coerce").astype("float64")
|
| 848 |
prices = prices[prices["sell_price_min"] > 0]
|
| 849 |
-
|
| 850 |
if prices.empty:
|
| 851 |
-
return pd.DataFrame({"Bilgi": ["Geçerli
|
| 852 |
|
| 853 |
buy = prices.loc[prices.groupby("item_id")["sell_price_min"].idxmin()][["item_id", "city", "sell_price_min"]]
|
| 854 |
buy.columns = ["item_id", "buy_city", "buy_price"]
|
| 855 |
-
|
| 856 |
sell = prices.loc[prices.groupby("item_id")["buy_price_max"].idxmax()][["item_id", "city", "buy_price_max"]]
|
| 857 |
sell.columns = ["item_id", "sell_city", "sell_price"]
|
| 858 |
|
| 859 |
arb = buy.merge(sell, on="item_id")
|
| 860 |
arb = arb[arb["sell_price"] > arb["buy_price"]].copy()
|
| 861 |
-
|
| 862 |
if arb.empty:
|
| 863 |
-
return pd.DataFrame({"Bilgi": ["Kârlı arbitraj
|
| 864 |
|
| 865 |
arb["brüt_kâr"] = arb["sell_price"] - arb["buy_price"]
|
| 866 |
-
arb["toplam_maliyet"] = (
|
| 867 |
-
arb["buy_price"] * SETUP_FEE_RATE
|
| 868 |
-
+ arb["sell_price"] * (SETUP_FEE_RATE + TRANSACTION_TAX_RATE)
|
| 869 |
-
)
|
| 870 |
arb["net_kâr"] = arb["brüt_kâr"] - arb["toplam_maliyet"]
|
| 871 |
arb["kâr_marjı_%"] = (arb["net_kâr"] / arb["buy_price"]) * 100
|
| 872 |
-
|
| 873 |
arb = arb[arb["net_kâr"] > 0].sort_values("kâr_marjı_%", ascending=False)
|
| 874 |
|
| 875 |
if arb.empty:
|
| 876 |
-
return pd.DataFrame({"Bilgi": ["
|
| 877 |
|
| 878 |
arb["Ürün"] = arb["item_id"].apply(item_display_name)
|
| 879 |
arb["Alış Şehri"] = arb["buy_city"]
|
|
@@ -883,54 +844,38 @@ def get_arbitrage_table():
|
|
| 883 |
arb["Net Kâr"] = arb["net_kâr"].round(0)
|
| 884 |
arb["Kâr Marjı %"] = arb["kâr_marjı_%"].round(1)
|
| 885 |
|
| 886 |
-
|
| 887 |
-
return arb[cols].head(50)
|
| 888 |
|
| 889 |
|
| 890 |
def get_model_performance_plot():
|
| 891 |
if pipeline.model is None:
|
| 892 |
-
return go.Figure().update_layout(
|
| 893 |
-
title="Model eğitilmedi - Lütfen 'Verileri Yenile' butonuna tıklayın",
|
| 894 |
-
template="plotly_dark"
|
| 895 |
-
)
|
| 896 |
-
|
| 897 |
try:
|
| 898 |
-
|
| 899 |
-
|
| 900 |
-
|
|
|
|
| 901 |
fig = px.bar(df_imp, x="Önem", y="Özellik", orientation="h",
|
| 902 |
title="Feature Importance (Top 20)", template="plotly_dark")
|
| 903 |
fig.update_layout(height=600)
|
| 904 |
return fig
|
| 905 |
except Exception as e:
|
| 906 |
-
return go.Figure().update_layout(title=f"
|
| 907 |
|
| 908 |
|
| 909 |
def get_backtest_outputs():
|
| 910 |
bt = pipeline.get_backtest()
|
| 911 |
-
|
| 912 |
if "error" in bt:
|
| 913 |
-
|
| 914 |
-
return pd.DataFrame({"Bilgi": [bt["error"]]}), empty_fig, pd.DataFrame()
|
| 915 |
|
| 916 |
results_df = pd.DataFrame(bt["results"])
|
| 917 |
-
|
| 918 |
fig = go.Figure()
|
| 919 |
fig.add_trace(go.Bar(name="Model MAE", x=results_df["split"], y=results_df["mae"], marker_color="cyan"))
|
| 920 |
fig.add_trace(go.Bar(name="Baseline MAE", x=results_df["split"], y=results_df["baseline_mae"], marker_color="gray"))
|
| 921 |
-
fig.update_layout(
|
| 922 |
-
|
| 923 |
-
xaxis_title="Split",
|
| 924 |
-
yaxis_title="MAE (log getiri)",
|
| 925 |
-
barmode="group",
|
| 926 |
-
template="plotly_dark",
|
| 927 |
-
)
|
| 928 |
-
|
| 929 |
-
if bt.get("simulation"):
|
| 930 |
-
sim_df = pd.DataFrame([bt["simulation"]])
|
| 931 |
-
else:
|
| 932 |
-
sim_df = pd.DataFrame()
|
| 933 |
|
|
|
|
| 934 |
return results_df, fig, sim_df
|
| 935 |
|
| 936 |
|
|
@@ -941,14 +886,14 @@ def list_csv_files() -> pd.DataFrame:
|
|
| 941 |
if fname.endswith(".csv"):
|
| 942 |
fpath = os.path.join(DATA_DIR, fname)
|
| 943 |
try:
|
| 944 |
-
|
| 945 |
-
|
| 946 |
-
|
|
|
|
|
|
|
| 947 |
except OSError:
|
| 948 |
continue
|
| 949 |
-
|
| 950 |
-
return pd.DataFrame(columns=["Dosya", "Boyut (KB)", "Son Değişiklik"])
|
| 951 |
-
return pd.DataFrame(rows)
|
| 952 |
|
| 953 |
|
| 954 |
def refresh_data_ui():
|
|
@@ -957,83 +902,49 @@ def refresh_data_ui():
|
|
| 957 |
if not pipeline.featured_df.empty:
|
| 958 |
pipeline.train_model()
|
| 959 |
except Exception as e:
|
| 960 |
-
pipeline.status_message = f"
|
| 961 |
-
print(f"Refresh hatası: {e}")
|
| 962 |
import traceback
|
| 963 |
traceback.print_exc()
|
| 964 |
|
| 965 |
try:
|
| 966 |
bt_table, bt_fig, sim_table = get_backtest_outputs()
|
| 967 |
-
except Exception
|
| 968 |
-
bt_table = pd.DataFrame(
|
| 969 |
bt_fig = go.Figure().update_layout(title="Backtest hatası", template="plotly_dark")
|
| 970 |
-
sim_table = pd.DataFrame()
|
| 971 |
-
|
| 972 |
-
try:
|
| 973 |
-
dash = get_dashboard_metrics()
|
| 974 |
-
except Exception as e:
|
| 975 |
-
dash = pd.DataFrame({"Bilgi": [f"Hata: {e}"]})
|
| 976 |
-
|
| 977 |
-
try:
|
| 978 |
-
ana = get_analysis_table()
|
| 979 |
-
except Exception as e:
|
| 980 |
-
ana = pd.DataFrame({"Bilgi": [f"Hata: {e}"]})
|
| 981 |
-
|
| 982 |
-
try:
|
| 983 |
-
pred = get_prediction_table()
|
| 984 |
-
except Exception as e:
|
| 985 |
-
pred = pd.DataFrame({"Bilgi": [f"Hata: {e}"]})
|
| 986 |
-
|
| 987 |
-
try:
|
| 988 |
-
arb = get_arbitrage_table()
|
| 989 |
-
except Exception as e:
|
| 990 |
-
arb = pd.DataFrame({"Bilgi": [f"Hata: {e}"]})
|
| 991 |
-
|
| 992 |
-
try:
|
| 993 |
-
perf = get_model_performance_plot()
|
| 994 |
-
except Exception as e:
|
| 995 |
-
perf = go.Figure().update_layout(title=f"Hata: {e}", template="plotly_dark")
|
| 996 |
|
| 997 |
-
|
| 998 |
-
|
| 999 |
-
|
| 1000 |
-
|
|
|
|
| 1001 |
|
| 1002 |
return (
|
| 1003 |
pipeline.status_message,
|
| 1004 |
-
|
| 1005 |
-
|
| 1006 |
-
|
| 1007 |
-
|
| 1008 |
-
|
| 1009 |
-
bt_table,
|
| 1010 |
-
|
| 1011 |
-
sim_table,
|
| 1012 |
-
csv_df,
|
| 1013 |
)
|
| 1014 |
|
| 1015 |
|
| 1016 |
def initial_load():
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
|
| 1020 |
-
|
| 1021 |
-
|
| 1022 |
-
try:
|
| 1023 |
-
csv_df = list_csv_files()
|
| 1024 |
-
except Exception:
|
| 1025 |
-
csv_df = pd.DataFrame(columns=["Dosya", "Boyut (KB)", "Son Değişiklik"])
|
| 1026 |
|
| 1027 |
return (
|
| 1028 |
-
|
| 1029 |
-
pd.DataFrame(),
|
| 1030 |
-
|
| 1031 |
pd.DataFrame(),
|
| 1032 |
-
go.Figure().update_layout(title="
|
| 1033 |
pd.DataFrame(),
|
| 1034 |
-
|
| 1035 |
-
pd.DataFrame(),
|
| 1036 |
-
csv_df,
|
| 1037 |
)
|
| 1038 |
|
| 1039 |
|
|
@@ -1050,10 +961,9 @@ with gr.Blocks(title="GOSHAWK ALBION MARKET AI", theme=gr.themes.Soft(primary_hu
|
|
| 1050 |
**Profesyonel Market Intelligence Dashboard** | Avrupa Sunucusu
|
| 1051 |
*LightGBM tabanlı 24 saatlik fiyat tahmini ve arbitraj analizi.*
|
| 1052 |
|
| 1053 |
-
**
|
| 1054 |
-
**Tahmin sınırı:** ±%15 (aşırı tahminler engellendi)
|
| 1055 |
|
| 1056 |
-
**⚠️ Risk Uyarısı:** Bu uygulama yatırım tavsiyesi değildir. Model tahminleri belirsizlik içerir ve kesin kazanç garantisi sunmaz.
|
| 1057 |
"""
|
| 1058 |
)
|
| 1059 |
|
|
@@ -1062,96 +972,56 @@ with gr.Blocks(title="GOSHAWK ALBION MARKET AI", theme=gr.themes.Soft(primary_hu
|
|
| 1062 |
status_text = gr.Textbox(label="Durum", interactive=False, scale=2, value=pipeline.status_message)
|
| 1063 |
|
| 1064 |
with gr.Tabs():
|
| 1065 |
-
|
| 1066 |
with gr.TabItem("📊 Dashboard"):
|
| 1067 |
-
dashboard_table = gr.Dataframe(label="Özet
|
| 1068 |
gr.Markdown(
|
| 1069 |
-
"###
|
| 1070 |
-
"-
|
| 1071 |
-
"-
|
| 1072 |
-
"- Tahmin
|
| 1073 |
-
"-
|
| 1074 |
"### Riskler\n"
|
| 1075 |
-
"- Model tahminleri **olasılıksaldır**
|
| 1076 |
-
"-
|
| 1077 |
-
"- Spread ve likidite düşükse gerçekleşme fiyatı
|
| 1078 |
)
|
| 1079 |
|
| 1080 |
with gr.TabItem("📦 100 Ürün Analizi"):
|
| 1081 |
-
analysis_table = gr.Dataframe(label="Ürün Analizi
|
| 1082 |
|
| 1083 |
with gr.TabItem("🔮 24 Saatlik Fiyat Tahmini"):
|
| 1084 |
-
prediction_table = gr.Dataframe(label="Tahminler
|
| 1085 |
-
gr.Markdown(
|
| 1086 |
-
"*Net getiri %2.5 setup + %8 vergi düşülerek hesaplanır. "
|
| 1087 |
-
"**Not** sütunundaki ⚠️ Belirsiz işareti, modelin sinyal üretmediği durumları gösterir. "
|
| 1088 |
-
"Bu tahminler garanti değildir.*"
|
| 1089 |
-
)
|
| 1090 |
|
| 1091 |
-
with gr.TabItem("🚚
|
| 1092 |
-
arbitrage_table = gr.Dataframe(label="Arbitraj Fırsatları
|
| 1093 |
-
gr.Markdown(
|
| 1094 |
-
"*Net kâr; alışta %2.5 setup fee, satışta %2.5 setup + %8 transaction tax düşülerek hesaplanmıştır. "
|
| 1095 |
-
"Seyahat maliyeti ve zaman riski dahil değildir.*"
|
| 1096 |
-
)
|
| 1097 |
|
| 1098 |
with gr.TabItem("📈 Model Performansı"):
|
| 1099 |
performance_plot = gr.Plot(label="Feature Importance")
|
| 1100 |
|
| 1101 |
-
with gr.TabItem("🧪
|
| 1102 |
backtest_table = gr.Dataframe(label="Split Sonuçları", interactive=False)
|
| 1103 |
backtest_plot = gr.Plot(label="MAE Karşılaştırması")
|
| 1104 |
-
simulation_table = gr.Dataframe(label="
|
| 1105 |
-
gr.Markdown(
|
| 1106 |
-
"*Walk-forward backtest kronolojik split kullanır; purge ve embargo uygulanır. "
|
| 1107 |
-
"Baseline 'fiyat değişmez' varsayımıdır. Simülasyon yalnızca modelin güven eşiğinin üzerinde "
|
| 1108 |
-
"pozitif getiri öngördüğü işlemleri içerir.*"
|
| 1109 |
-
)
|
| 1110 |
|
| 1111 |
with gr.TabItem("🗄️ Veri Yönetimi"):
|
| 1112 |
-
gr.Markdown(
|
| 1113 |
-
|
| 1114 |
-
|
| 1115 |
-
"**Endpoint'ler:** `/stats/prices/` (anlık), `/stats/history/` (tarihsel)\n"
|
| 1116 |
-
"**Model dosyası:** `gos_hawk_data/models/gos_hawk_model.pkl`\n"
|
| 1117 |
-
)
|
| 1118 |
-
csv_files = gr.Dataframe(
|
| 1119 |
-
label="Kayıtlı CSV Dosyaları (Yenile butonuna basınca güncellenir)",
|
| 1120 |
-
interactive=False,
|
| 1121 |
-
headers=["Dosya", "Boyut (KB)", "Son Değişiklik"],
|
| 1122 |
-
)
|
| 1123 |
|
| 1124 |
refresh_btn.click(
|
| 1125 |
fn=refresh_data_ui,
|
| 1126 |
inputs=[],
|
| 1127 |
-
outputs=[
|
| 1128 |
-
|
| 1129 |
-
|
| 1130 |
-
analysis_table,
|
| 1131 |
-
prediction_table,
|
| 1132 |
-
arbitrage_table,
|
| 1133 |
-
performance_plot,
|
| 1134 |
-
backtest_table,
|
| 1135 |
-
backtest_plot,
|
| 1136 |
-
simulation_table,
|
| 1137 |
-
csv_files,
|
| 1138 |
-
],
|
| 1139 |
)
|
| 1140 |
|
| 1141 |
demo.load(
|
| 1142 |
fn=initial_load,
|
| 1143 |
inputs=[],
|
| 1144 |
-
outputs=[
|
| 1145 |
-
|
| 1146 |
-
analysis_table,
|
| 1147 |
-
prediction_table,
|
| 1148 |
-
arbitrage_table,
|
| 1149 |
-
performance_plot,
|
| 1150 |
-
backtest_table,
|
| 1151 |
-
backtest_plot,
|
| 1152 |
-
simulation_table,
|
| 1153 |
-
csv_files,
|
| 1154 |
-
],
|
| 1155 |
)
|
| 1156 |
|
| 1157 |
|
|
|
|
| 35 |
API_BASE = "https://europe.albion-online-data.com/api/v2"
|
| 36 |
|
| 37 |
CITIES = [
|
| 38 |
+
"Bridgewatch", "Fort Sterling", "Lymhurst",
|
| 39 |
+
"Martlock", "Thetford", "Caerleon",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
]
|
| 41 |
|
| 42 |
_ITEM_POOL = [
|
|
|
|
| 90 |
|
| 91 |
DATA_DIR = "gos_hawk_data"
|
| 92 |
os.makedirs(DATA_DIR, exist_ok=True)
|
|
|
|
| 93 |
MODEL_DIR = os.path.join(DATA_DIR, "models")
|
| 94 |
os.makedirs(MODEL_DIR, exist_ok=True)
|
| 95 |
MODEL_PATH = os.path.join(MODEL_DIR, "gos_hawk_model.pkl")
|
| 96 |
|
|
|
|
|
|
|
| 97 |
MIN_TRAIN_SAMPLES = 150
|
|
|
|
| 98 |
REQUEST_TIMEOUT = 30
|
| 99 |
MAX_RETRIES = 3
|
| 100 |
RETRY_BACKOFF = 2
|
|
|
|
| 101 |
HISTORY_TIME_SCALE = 6
|
| 102 |
|
| 103 |
+
# KRİTİK: Hedef ölçeği — log getiri * 10000 = baz puan
|
| 104 |
+
# Bu, modelin 0.1-1 aralığında öğrenmesini sağlar → stabil
|
| 105 |
+
TARGET_SCALE = 10000.0
|
| 106 |
|
| 107 |
+
# Tahmin clip (log getiri, ±%15)
|
| 108 |
MAX_LOG_RETURN = 0.15
|
| 109 |
MIN_LOG_RETURN = -0.15
|
| 110 |
|
| 111 |
+
# Belirsizlik eşiği (log getiri)
|
| 112 |
+
LOW_CONFIDENCE_THRESHOLD = 0.01
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
|
| 114 |
# ---------------------------------------------------------------------------
|
| 115 |
# PUBLIC API ÇAĞRILARI
|
|
|
|
| 118 |
|
| 119 |
def safe_request(url: str, params: Optional[dict] = None) -> Optional[list]:
|
| 120 |
headers = {
|
| 121 |
+
"User-Agent": "GOSHAWK-ALBION-MARKET-AI/1.3 (public-api-client)",
|
| 122 |
"Accept": "application/json",
|
| 123 |
}
|
| 124 |
for attempt in range(MAX_RETRIES):
|
|
|
|
| 132 |
time.sleep(wait)
|
| 133 |
continue
|
| 134 |
elif resp.status_code >= 500:
|
|
|
|
| 135 |
time.sleep(RETRY_BACKOFF * (attempt + 1))
|
| 136 |
continue
|
| 137 |
else:
|
|
|
|
| 152 |
chunk_size = 50
|
| 153 |
for i in range(0, len(item_ids), chunk_size):
|
| 154 |
chunk = item_ids[i:i+chunk_size]
|
| 155 |
+
url = f"{API_BASE}/stats/prices/{','.join(chunk)}.json"
|
| 156 |
+
params = {"locations": ",".join(cities), "qualities": qualities}
|
|
|
|
|
|
|
| 157 |
data = safe_request(url, params)
|
| 158 |
if data:
|
| 159 |
all_data.extend(data)
|
|
|
|
| 165 |
for col in ["sell_price_min", "sell_price_max", "buy_price_min", "buy_price_max"]:
|
| 166 |
if col in df.columns:
|
| 167 |
df[col] = pd.to_numeric(df[col], errors="coerce").astype("float64")
|
|
|
|
|
|
|
|
|
|
| 168 |
return df
|
| 169 |
|
| 170 |
|
|
|
|
| 173 |
chunk_size = 30
|
| 174 |
for i in range(0, len(item_ids), chunk_size):
|
| 175 |
chunk = item_ids[i:i+chunk_size]
|
| 176 |
+
url = f"{API_BASE}/stats/history/{','.join(chunk)}.json"
|
| 177 |
+
params = {"locations": ",".join(cities), "qualities": qualities, "time-scale": time_scale}
|
|
|
|
|
|
|
| 178 |
data = safe_request(url, params)
|
| 179 |
if data:
|
| 180 |
for entry in data:
|
| 181 |
item_id = entry.get("item_id")
|
| 182 |
loc = entry.get("location")
|
|
|
|
| 183 |
for point in entry.get("data", []):
|
| 184 |
all_records.append({
|
| 185 |
"item_id": item_id,
|
| 186 |
"city": loc,
|
|
|
|
| 187 |
"timestamp": point.get("timestamp"),
|
| 188 |
"avg_price": point.get("avg_price"),
|
| 189 |
"item_count": point.get("item_count", 0),
|
|
|
|
| 202 |
def check_api_health() -> Tuple[bool, str]:
|
| 203 |
try:
|
| 204 |
url = f"{API_BASE}/stats/prices/T4_BAG.json"
|
| 205 |
+
data = safe_request(url, {"locations": "Bridgewatch", "qualities": "1"})
|
| 206 |
+
return (True, "OK") if data is not None else (False, "Yanıt yok")
|
|
|
|
|
|
|
|
|
|
| 207 |
except Exception as e:
|
| 208 |
+
return False, str(e)
|
| 209 |
|
| 210 |
|
| 211 |
def clean_market_data(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
|
| 213 |
return df
|
| 214 |
df = df.copy()
|
| 215 |
df["avg_price"] = pd.to_numeric(df["avg_price"], errors="coerce").astype("float64")
|
| 216 |
+
df["item_count"] = pd.to_numeric(df["item_count"], errors="coerce").fillna(0).astype("float64")
|
|
|
|
| 217 |
df = df[df["avg_price"] > 0].copy()
|
| 218 |
|
| 219 |
+
for item in df["item_id"].unique():
|
| 220 |
+
mask = df["item_id"] == item
|
| 221 |
+
if mask.sum() < 10:
|
| 222 |
+
continue
|
| 223 |
+
lower = df.loc[mask, "avg_price"].quantile(0.005)
|
| 224 |
+
upper = df.loc[mask, "avg_price"].quantile(0.995)
|
| 225 |
+
if pd.notna(lower) and pd.notna(upper) and upper > lower:
|
| 226 |
+
df.loc[mask, "avg_price"] = df.loc[mask, "avg_price"].clip(lower, upper)
|
|
|
|
| 227 |
|
| 228 |
+
return df.sort_values(["item_id", "city", "timestamp"]).reset_index(drop=True)
|
|
|
|
| 229 |
|
| 230 |
|
| 231 |
def save_to_csv(df: pd.DataFrame, name: str) -> str:
|
|
|
|
| 250 |
gain = delta.clip(lower=0).rolling(period).mean()
|
| 251 |
loss = (-delta.clip(upper=0)).rolling(period).mean()
|
| 252 |
rs = gain / loss.replace(0, np.nan)
|
| 253 |
+
return (100 - (100 / (1 + rs))).fillna(50)
|
|
|
|
| 254 |
|
| 255 |
|
| 256 |
def build_features(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
|
| 294 |
city_avg = df.groupby(["item_id", "timestamp"])["avg_price"].transform("mean")
|
| 295 |
df["city_price_deviation"] = (df["avg_price"] - city_avg) / city_avg.replace(0, np.nan)
|
| 296 |
|
| 297 |
+
# KRİTİK: hedef ölçeği = log getiri * 10000 (baz puan)
|
| 298 |
+
df["target_return_24h"] = df.groupby(group_cols)["log_price"].shift(-4) * TARGET_SCALE
|
| 299 |
|
| 300 |
return df
|
| 301 |
|
|
|
|
| 317 |
|
| 318 |
|
| 319 |
def prepare_model_data(featured_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame]:
|
| 320 |
+
"""
|
| 321 |
+
Feature engineering çıktısını modele hazırlar.
|
| 322 |
+
KRİTİK: Fiyat-benzeri feature'lar log1p ile normalize edilir.
|
| 323 |
+
"""
|
| 324 |
if featured_df.empty:
|
| 325 |
return pd.DataFrame(), pd.Series(dtype=float), pd.DataFrame()
|
| 326 |
|
|
|
|
| 330 |
|
| 331 |
df = df.dropna(subset=["target_return_24h", "lag_1h", "lag_24h"]).copy()
|
| 332 |
|
| 333 |
+
# KRİTİK: Fiyat-benzeri feature'ları log ölçeğine çevir (binlerce silver → ~7-10)
|
| 334 |
+
price_like_cols = [
|
| 335 |
+
"lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
|
| 336 |
+
"ma_6h", "ma_12h", "ma_24h", "ma_48h",
|
| 337 |
+
]
|
| 338 |
+
for col in price_like_cols:
|
| 339 |
+
if col in df.columns:
|
| 340 |
+
df[col] = np.log1p(df[col].clip(lower=1.0))
|
| 341 |
+
|
| 342 |
+
# Uzun pencereleri doldur
|
| 343 |
if "lag_48h" in df.columns:
|
| 344 |
df["lag_48h"] = df["lag_48h"].fillna(df["lag_24h"])
|
| 345 |
if "ma_48h" in df.columns:
|
| 346 |
df["ma_48h"] = df["ma_48h"].fillna(df["ma_24h"])
|
| 347 |
if "ma_24h" in df.columns:
|
| 348 |
df["ma_24h"] = df["ma_24h"].fillna(df["lag_24h"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
|
| 350 |
for chg in ["chg_1h", "chg_6h", "chg_12h", "chg_24h"]:
|
| 351 |
if chg in df.columns:
|
|
|
|
| 359 |
|
| 360 |
model_df = df.dropna(subset=FEATURE_COLS + ["target_return_24h"]).copy()
|
| 361 |
|
| 362 |
+
# KRİTİK: Target zaten baz puan (×10000). Şimdi ±1500 bp (≈ ±0.15 log) sınırı
|
| 363 |
+
target_min = MIN_LOG_RETURN * TARGET_SCALE
|
| 364 |
+
target_max = MAX_LOG_RETURN * TARGET_SCALE
|
| 365 |
+
model_df["target_return_24h"] = model_df["target_return_24h"].clip(target_min, target_max)
|
|
|
|
|
|
|
|
|
|
| 366 |
|
| 367 |
X = model_df[FEATURE_COLS].astype("float64")
|
| 368 |
y = model_df["target_return_24h"].astype("float64")
|
| 369 |
+
meta = model_df[["item_id", "city", "timestamp", "avg_price"]].copy()
|
| 370 |
+
meta["target_log_return"] = model_df["target_return_24h"] / TARGET_SCALE
|
| 371 |
|
| 372 |
return X, y, meta
|
| 373 |
|
|
|
|
| 377 |
print(f"Yetersiz veri: {len(X)} < {MIN_TRAIN_SAMPLES}")
|
| 378 |
return None
|
| 379 |
|
| 380 |
+
# Hedef baz puan ölçeğinde → standart sapma ~100-300 bp
|
| 381 |
+
y_std = y.std()
|
| 382 |
+
if y_std < 1e-3:
|
| 383 |
+
print(f"⚠️ Hedef standart sapması çok düşük: {y_std}. Model öğrenemez.")
|
| 384 |
+
return None
|
| 385 |
+
|
| 386 |
split_idx = int(len(X) * 0.8)
|
| 387 |
X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
|
| 388 |
y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
|
| 389 |
|
| 390 |
+
# Regularizasyon DENGELİ — overfit değil, underfit de değil
|
| 391 |
model = lgb.LGBMRegressor(
|
| 392 |
objective="regression",
|
| 393 |
metric="mae",
|
| 394 |
boosting_type="gbdt",
|
| 395 |
+
num_leaves=31,
|
| 396 |
+
max_depth=8,
|
| 397 |
+
min_child_samples=20,
|
| 398 |
+
learning_rate=0.05,
|
| 399 |
+
n_estimators=600,
|
| 400 |
+
feature_fraction=0.8,
|
| 401 |
+
bagging_fraction=0.8,
|
| 402 |
bagging_freq=5,
|
| 403 |
+
reg_alpha=0.0,
|
| 404 |
+
reg_lambda=0.1,
|
| 405 |
verbose=-1,
|
| 406 |
random_state=42,
|
| 407 |
n_jobs=-1,
|
| 408 |
)
|
| 409 |
|
| 410 |
callbacks = [
|
| 411 |
+
lgb.early_stopping(stopping_rounds=50, verbose=False),
|
| 412 |
lgb.log_evaluation(period=0),
|
| 413 |
]
|
| 414 |
|
|
|
|
| 419 |
callbacks=callbacks,
|
| 420 |
)
|
| 421 |
|
| 422 |
+
# Öğrendi mi kontrolü
|
| 423 |
+
val_preds = model.predict(X_val)
|
| 424 |
+
if np.std(val_preds) < 1e-3:
|
| 425 |
+
print(f"⚠️ Validation tahminleri neredeyse sabit (std={np.std(val_preds):.6f}). Model öğrenmedi.")
|
| 426 |
+
|
| 427 |
return model
|
| 428 |
|
| 429 |
|
| 430 |
def predict_24h_returns(model: lgb.LGBMRegressor, latest_features: pd.DataFrame) -> pd.DataFrame:
|
| 431 |
"""
|
| 432 |
+
Ham tahmin baz puan cinsinden → log getiriye çevir → clip → exp.
|
|
|
|
|
|
|
|
|
|
| 433 |
"""
|
| 434 |
if latest_features.empty or model is None:
|
| 435 |
return pd.DataFrame()
|
|
|
|
| 439 |
print(f"Eksik özellikler: {missing}")
|
| 440 |
return pd.DataFrame()
|
| 441 |
|
| 442 |
+
# KRİTİK: predict öncesi de aynı log1p dönüşümü uygula
|
| 443 |
+
X_pred = latest_features.copy()
|
| 444 |
+
price_like_cols = [
|
| 445 |
+
"lag_1h", "lag_3h", "lag_6h", "lag_12h", "lag_24h", "lag_48h",
|
| 446 |
+
"ma_6h", "ma_12h", "ma_24h", "ma_48h",
|
| 447 |
+
]
|
| 448 |
+
for col in price_like_cols:
|
| 449 |
+
if col in X_pred.columns:
|
| 450 |
+
X_pred[col] = np.log1p(X_pred[col].clip(lower=1.0))
|
| 451 |
+
|
| 452 |
+
for chg in ["chg_1h", "chg_6h", "chg_12h", "chg_24h"]:
|
| 453 |
+
if chg in X_pred.columns:
|
| 454 |
+
X_pred[chg] = X_pred[chg].fillna(0.0)
|
| 455 |
+
|
| 456 |
+
X_pred["rsi_14"] = X_pred["rsi_14"].fillna(50.0)
|
| 457 |
+
X_pred["volatility_24h"] = X_pred["volatility_24h"].fillna(0.0)
|
| 458 |
+
X_pred["volume_ma_24h"] = X_pred["volume_ma_24h"].fillna(0.0)
|
| 459 |
+
X_pred["volume_ratio"] = X_pred["volume_ratio"].fillna(1.0)
|
| 460 |
+
X_pred["city_price_deviation"] = X_pred["city_price_deviation"].fillna(0.0)
|
| 461 |
+
|
| 462 |
+
X_ready = X_pred[FEATURE_COLS].astype("float64").dropna()
|
| 463 |
+
if X_ready.empty:
|
| 464 |
return pd.DataFrame()
|
| 465 |
|
| 466 |
+
raw_preds_bp = model.predict(X_ready) # baz puan
|
| 467 |
+
raw_preds_log = raw_preds_bp / TARGET_SCALE # log getiri
|
| 468 |
|
| 469 |
+
# Tahmin çeşitliliği kontrolü
|
| 470 |
+
pred_std = np.std(raw_preds_log)
|
| 471 |
+
if pred_std < 1e-5:
|
| 472 |
+
print(f"⚠️ UYARI: Tüm tahminler aynı (std={pred_std:.8f}).")
|
| 473 |
|
| 474 |
+
# Aşırı değerleri clip'le
|
| 475 |
+
clipped_log = np.clip(raw_preds_log, MIN_LOG_RETURN, MAX_LOG_RETURN)
|
| 476 |
|
| 477 |
+
pred_return_pct = (np.exp(clipped_log) - 1) * 100
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 478 |
|
| 479 |
+
result = X_pred.loc[X_ready.index, ["item_id", "city", "avg_price", "item_count",
|
| 480 |
+
"volatility_24h", "rsi_14", "volume_ratio"]].copy()
|
| 481 |
+
result["raw_log_pred"] = raw_preds_log
|
| 482 |
+
result["clipped_log_pred"] = clipped_log
|
| 483 |
+
result["predicted_log_return_24h"] = clipped_log
|
| 484 |
+
result["predicted_return_pct"] = pred_return_pct
|
| 485 |
+
result["predicted_price_24h"] = result["avg_price"] * np.exp(clipped_log)
|
| 486 |
+
result["is_uncertain"] = np.abs(clipped_log) < LOW_CONFIDENCE_THRESHOLD
|
| 487 |
|
| 488 |
return result
|
| 489 |
|
|
|
|
| 499 |
|
| 500 |
df = featured_df.sort_values("timestamp").copy()
|
| 501 |
unique_times = df["timestamp"].sort_values().unique()
|
|
|
|
| 502 |
if len(unique_times) < 50:
|
| 503 |
+
return {"error": "Yeterli zaman noktası yok."}
|
| 504 |
|
| 505 |
test_size = len(unique_times) // (n_splits + 1)
|
| 506 |
results = []
|
|
|
|
| 511 |
test_end_idx = test_start_idx + test_size
|
| 512 |
if test_end_idx > len(unique_times):
|
| 513 |
break
|
|
|
|
| 514 |
train_end_idx = test_start_idx - 8
|
| 515 |
if train_end_idx <= 0:
|
| 516 |
continue
|
|
|
|
| 531 |
continue
|
| 532 |
|
| 533 |
model = lgb.LGBMRegressor(
|
| 534 |
+
objective="regression", metric="mae",
|
| 535 |
+
num_leaves=31, max_depth=8, min_child_samples=20,
|
| 536 |
+
learning_rate=0.05, n_estimators=400,
|
| 537 |
+
reg_lambda=0.1, verbose=-1, random_state=42, n_jobs=-1,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 538 |
)
|
| 539 |
model.fit(X_train, y_train)
|
| 540 |
|
| 541 |
+
preds_bp = model.predict(X_test)
|
| 542 |
+
preds = np.clip(preds_bp / TARGET_SCALE, MIN_LOG_RETURN, MAX_LOG_RETURN)
|
| 543 |
+
y_test_log = y_test / TARGET_SCALE
|
| 544 |
|
| 545 |
+
mae = mean_absolute_error(y_test_log, preds)
|
| 546 |
+
rmse = np.sqrt(mean_squared_error(y_test_log, preds))
|
| 547 |
+
direction_correct = np.mean((y_test_log > 0) == (preds > 0))
|
| 548 |
|
| 549 |
+
baseline_preds = np.zeros_like(y_test_log)
|
| 550 |
+
baseline_mae = mean_absolute_error(y_test_log, baseline_preds)
|
| 551 |
+
baseline_rmse = np.sqrt(mean_squared_error(y_test_log, baseline_preds))
|
| 552 |
|
| 553 |
results.append({
|
| 554 |
"split": i + 1,
|
|
|
|
| 564 |
|
| 565 |
test_meta = meta_test.copy()
|
| 566 |
test_meta["predicted_return"] = preds
|
| 567 |
+
test_meta["actual_return"] = y_test_log.values
|
| 568 |
all_predictions.append(test_meta)
|
| 569 |
|
| 570 |
if not results:
|
| 571 |
+
return {"error": "Backtest tamamlanamadı."}
|
| 572 |
|
| 573 |
if all_predictions:
|
| 574 |
sim_df = pd.concat(all_predictions)
|
|
|
|
| 578 |
sim_trades = sim_df[sim_df["trade_signal"]].copy()
|
| 579 |
|
| 580 |
if len(sim_trades) > 0:
|
|
|
|
|
|
|
| 581 |
simulation = {
|
| 582 |
"total_trades": int(len(sim_trades)),
|
| 583 |
+
"avg_net_return_per_trade_pct": round(float(sim_trades["net_return"].mean() * 100), 4),
|
| 584 |
+
"win_rate_pct": round(float((sim_trades["net_return"] > 0).mean() * 100), 2),
|
| 585 |
"profitable_trades": int((sim_trades["net_return"] > 0).sum()),
|
| 586 |
}
|
| 587 |
else:
|
|
|
|
| 589 |
else:
|
| 590 |
simulation = {}
|
| 591 |
|
| 592 |
+
return {"results": results, "simulation": simulation, "n_splits_completed": len(results)}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
|
| 594 |
|
| 595 |
# ---------------------------------------------------------------------------
|
|
|
|
| 606 |
self.last_update = None
|
| 607 |
self.status_message = "Veri yüklenmedi. 'Verileri Yenile' butonuna basın."
|
| 608 |
self._backtest_cache = None
|
|
|
|
| 609 |
|
| 610 |
if _HAS_JOBLIB and os.path.exists(MODEL_PATH):
|
| 611 |
try:
|
| 612 |
self.model = joblib.load(MODEL_PATH)
|
| 613 |
+
self.status_message = "Kayıtlı model yüklendi."
|
|
|
|
| 614 |
except Exception as e:
|
| 615 |
print(f"Model yükleme hatası: {e}")
|
| 616 |
|
| 617 |
+
def fetch_data(self, item_ids=None, cities=None):
|
| 618 |
+
item_ids = item_ids or ITEM_IDS
|
| 619 |
+
cities = cities or CITIES
|
|
|
|
|
|
|
| 620 |
|
| 621 |
+
self.status_message = "Public API'den veri çekiliyor..."
|
| 622 |
ok, msg = check_api_health()
|
| 623 |
if not ok:
|
| 624 |
self.status_message = f"API erişilemiyor: {msg}"
|
|
|
|
| 632 |
return
|
| 633 |
|
| 634 |
if hist.empty:
|
| 635 |
+
self.status_message = "Tarihsel veri alınamadı."
|
| 636 |
return
|
| 637 |
|
| 638 |
self.history_df = clean_market_data(hist)
|
| 639 |
self.prices_df = prices
|
| 640 |
+
self.status_message = f"✓ {len(self.history_df)} satır tarihsel, {len(self.prices_df)} anlık."
|
|
|
|
|
|
|
|
|
|
| 641 |
|
| 642 |
save_to_csv(self.history_df, "history")
|
| 643 |
save_to_csv(self.prices_df, "prices")
|
|
|
|
| 652 |
|
| 653 |
X, y, _ = prepare_model_data(self.featured_df)
|
| 654 |
if len(X) < MIN_TRAIN_SAMPLES:
|
| 655 |
+
self.status_message = f"Yetersiz örnek: {len(X)} (min {MIN_TRAIN_SAMPLES})"
|
| 656 |
+
return False
|
| 657 |
+
|
| 658 |
+
# Hedef dağılımı kontrolü
|
| 659 |
+
y_std = y.std()
|
| 660 |
+
if y_std < 1e-3:
|
| 661 |
+
self.status_message = f"Hedef varyansı çok düşük (std={y_std:.4f}). Model öğrenemez."
|
| 662 |
return False
|
| 663 |
|
| 664 |
self.model = train_global_model(X, y)
|
|
|
|
| 666 |
self.status_message = "Model eğitimi başarısız."
|
| 667 |
return False
|
| 668 |
|
| 669 |
+
# Tahmin çeşitliliği kontrolü (eğitim setinde)
|
| 670 |
+
sample_preds = self.model.predict(X.head(500))
|
| 671 |
+
pred_std = np.std(sample_preds)
|
| 672 |
+
if pred_std < 1.0: # bp cinsinden
|
| 673 |
+
self.status_message = f"⚠️ Model öğrenmedi (std={pred_std:.4f} bp). Daha fazla veri gerekli."
|
| 674 |
+
else:
|
| 675 |
+
self.status_message = f"Model eğitildi ({len(X)} örnek, pred_std={pred_std:.1f} bp)."
|
| 676 |
+
|
| 677 |
if _HAS_JOBLIB:
|
| 678 |
try:
|
| 679 |
joblib.dump(self.model, MODEL_PATH)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
except Exception as e:
|
| 681 |
+
print(f"Kaydetme hatası: {e}")
|
|
|
|
|
|
|
| 682 |
|
| 683 |
self.last_update = dt.datetime.now()
|
| 684 |
return True
|
|
|
|
| 706 |
else:
|
| 707 |
preds["spread"] = np.nan
|
| 708 |
|
|
|
|
| 709 |
vol_score = (1.0 - preds["volatility_24h"].clip(0, 0.2) / 0.2).fillna(0.5)
|
| 710 |
rsi_penalty = ((preds["rsi_14"] - 50).abs() / 50).clip(0, 1).fillna(0.5) * 0.2
|
| 711 |
vol_ratio_score = preds["volume_ratio"].clip(0, 2).fillna(1.0) / 2
|
| 712 |
base_conf = vol_score * 0.6 + vol_ratio_score * 0.4 - rsi_penalty
|
|
|
|
| 713 |
certainty_penalty = preds["is_uncertain"].astype(float) * 0.4
|
| 714 |
preds["confidence"] = (base_conf - certainty_penalty).clip(0, 1).fillna(0.3)
|
| 715 |
|
|
|
|
| 728 |
|
| 729 |
|
| 730 |
# ---------------------------------------------------------------------------
|
| 731 |
+
# GRADIO YARDIMCILARI
|
| 732 |
# ---------------------------------------------------------------------------
|
| 733 |
|
| 734 |
|
| 735 |
def get_dashboard_metrics():
|
| 736 |
if pipeline.featured_df.empty:
|
| 737 |
return pd.DataFrame({
|
| 738 |
+
"Metrik": ["Durum", "Veri Kaynağı", "Model"],
|
| 739 |
+
"Değer": ["Veri yok.", "Public AODP API (europe)", "Yok" if pipeline.model is None else "Yüklü"],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 740 |
})
|
| 741 |
|
| 742 |
df = pipeline.featured_df
|
| 743 |
metrics = {
|
| 744 |
+
"Veri Kaynağı": "Public AODP API (europe)",
|
| 745 |
+
"Time-scale": f"{HISTORY_TIME_SCALE}h",
|
| 746 |
"Toplam Gözlem": len(df),
|
| 747 |
"Ürün Sayısı": df["item_id"].nunique(),
|
| 748 |
"Şehir Sayısı": df["city"].nunique(),
|
| 749 |
"Tarih Aralığı": f"{df['timestamp'].min().date()} - {df['timestamp'].max().date()}",
|
| 750 |
"Ortalama Fiyat": f"{df['avg_price'].mean():.0f} silver",
|
|
|
|
| 751 |
"Model Durumu": "Eğitildi" if pipeline.model else "Eğitilmedi",
|
| 752 |
"Tahmin Sınırı": f"±%{MAX_LOG_RETURN * 100:.0f}",
|
| 753 |
+
"Hedef Ölçeği": f"baz puan (×{TARGET_SCALE:.0f})",
|
| 754 |
"Son Güncelleme": pipeline.last_update.strftime("%Y-%m-%d %H:%M") if pipeline.last_update else "—",
|
| 755 |
}
|
| 756 |
return pd.DataFrame(list(metrics.items()), columns=["Metrik", "Değer"])
|
|
|
|
| 776 |
latest["Mevcut Fiyat"] = latest["avg_price"].round(0)
|
| 777 |
latest["Hacim"] = latest["item_count"].fillna(0).astype(int)
|
| 778 |
latest["Spread"] = (latest["sell_price_min"] - latest["buy_price_max"]).round(0)
|
| 779 |
+
latest["Likidite"] = pd.cut(latest["item_count"], bins=[-1, 10, 50, 200, np.inf],
|
| 780 |
+
labels=["Düşük", "Orta", "İyi", "Yüksek"]).astype(str)
|
|
|
|
|
|
|
|
|
|
| 781 |
|
| 782 |
+
return latest[["Ürün", "Şehir", "Mevcut Fiyat", "Hacim", "Spread", "Likidite"]].sort_values("Hacim", ascending=False).head(100)
|
|
|
|
| 783 |
|
| 784 |
|
| 785 |
def get_prediction_table():
|
| 786 |
preds = pipeline.get_latest_predictions()
|
| 787 |
if preds.empty:
|
| 788 |
+
return pd.DataFrame({"Bilgi": ["Tahmin üretilemedi."]})
|
|
|
|
|
|
|
| 789 |
|
| 790 |
preds = preds.copy()
|
| 791 |
preds["Ürün"] = preds["item_id"].apply(item_display_name)
|
|
|
|
| 796 |
preds["Güvenilirlik"] = (preds["confidence"] * 100).round(0).astype(int).astype(str) + "%"
|
| 797 |
preds["Spread"] = preds["spread"].round(0)
|
| 798 |
preds["Net Getiri % (maliyet sonrası)"] = preds["net_return_pct"].round(2)
|
| 799 |
+
preds["Likidite"] = pd.cut(preds["item_count"], bins=[-1, 10, 50, 200, np.inf],
|
| 800 |
+
labels=["Düşük", "Orta", "İyi", "Yüksek"]).astype(str)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 801 |
preds["Not"] = np.where(preds["is_uncertain"], "⚠️ Belirsiz", "")
|
| 802 |
|
| 803 |
+
cols = ["Ürün", "Şehir", "Mevcut Fiyat", "Tahmini Fiyat (24s)",
|
| 804 |
+
"Tahmini Deği��im %", "Güvenilirlik", "Spread", "Likidite",
|
| 805 |
+
"Net Getiri % (maliyet sonrası)", "Not"]
|
|
|
|
|
|
|
| 806 |
return preds[cols].sort_values("Tahmini Değişim %", ascending=False)
|
| 807 |
|
| 808 |
|
| 809 |
def get_arbitrage_table():
|
| 810 |
if pipeline.prices_df.empty:
|
| 811 |
+
return pd.DataFrame({"Bilgi": ["Anlık fiyat yok."]})
|
| 812 |
|
| 813 |
prices = pipeline.prices_df.copy()
|
| 814 |
prices["sell_price_min"] = pd.to_numeric(prices["sell_price_min"], errors="coerce").astype("float64")
|
| 815 |
prices["buy_price_max"] = pd.to_numeric(prices["buy_price_max"], errors="coerce").astype("float64")
|
| 816 |
prices = prices[prices["sell_price_min"] > 0]
|
|
|
|
| 817 |
if prices.empty:
|
| 818 |
+
return pd.DataFrame({"Bilgi": ["Geçerli fiyat yok."]})
|
| 819 |
|
| 820 |
buy = prices.loc[prices.groupby("item_id")["sell_price_min"].idxmin()][["item_id", "city", "sell_price_min"]]
|
| 821 |
buy.columns = ["item_id", "buy_city", "buy_price"]
|
|
|
|
| 822 |
sell = prices.loc[prices.groupby("item_id")["buy_price_max"].idxmax()][["item_id", "city", "buy_price_max"]]
|
| 823 |
sell.columns = ["item_id", "sell_city", "sell_price"]
|
| 824 |
|
| 825 |
arb = buy.merge(sell, on="item_id")
|
| 826 |
arb = arb[arb["sell_price"] > arb["buy_price"]].copy()
|
|
|
|
| 827 |
if arb.empty:
|
| 828 |
+
return pd.DataFrame({"Bilgi": ["Kârlı arbitraj yok."]})
|
| 829 |
|
| 830 |
arb["brüt_kâr"] = arb["sell_price"] - arb["buy_price"]
|
| 831 |
+
arb["toplam_maliyet"] = arb["buy_price"] * SETUP_FEE_RATE + arb["sell_price"] * (SETUP_FEE_RATE + TRANSACTION_TAX_RATE)
|
|
|
|
|
|
|
|
|
|
| 832 |
arb["net_kâr"] = arb["brüt_kâr"] - arb["toplam_maliyet"]
|
| 833 |
arb["kâr_marjı_%"] = (arb["net_kâr"] / arb["buy_price"]) * 100
|
|
|
|
| 834 |
arb = arb[arb["net_kâr"] > 0].sort_values("kâr_marjı_%", ascending=False)
|
| 835 |
|
| 836 |
if arb.empty:
|
| 837 |
+
return pd.DataFrame({"Bilgi": ["Maliyetler sonrası kârlı arbitraj yok."]})
|
| 838 |
|
| 839 |
arb["Ürün"] = arb["item_id"].apply(item_display_name)
|
| 840 |
arb["Alış Şehri"] = arb["buy_city"]
|
|
|
|
| 844 |
arb["Net Kâr"] = arb["net_kâr"].round(0)
|
| 845 |
arb["Kâr Marjı %"] = arb["kâr_marjı_%"].round(1)
|
| 846 |
|
| 847 |
+
return arb[["Ürün", "Alış Şehri", "Satış Şehri", "Alış Fiyatı", "Satış Fiyatı", "Net Kâr", "Kâr Marjı %"]].head(50)
|
|
|
|
| 848 |
|
| 849 |
|
| 850 |
def get_model_performance_plot():
|
| 851 |
if pipeline.model is None:
|
| 852 |
+
return go.Figure().update_layout(title="Model eğitilmedi", template="plotly_dark")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 853 |
try:
|
| 854 |
+
df_imp = pd.DataFrame({
|
| 855 |
+
"Özellik": pipeline.model.feature_name_,
|
| 856 |
+
"Önem": pipeline.model.feature_importances_,
|
| 857 |
+
}).sort_values("Önem", ascending=True).tail(20)
|
| 858 |
fig = px.bar(df_imp, x="Önem", y="Özellik", orientation="h",
|
| 859 |
title="Feature Importance (Top 20)", template="plotly_dark")
|
| 860 |
fig.update_layout(height=600)
|
| 861 |
return fig
|
| 862 |
except Exception as e:
|
| 863 |
+
return go.Figure().update_layout(title=f"Hata: {e}", template="plotly_dark")
|
| 864 |
|
| 865 |
|
| 866 |
def get_backtest_outputs():
|
| 867 |
bt = pipeline.get_backtest()
|
|
|
|
| 868 |
if "error" in bt:
|
| 869 |
+
return pd.DataFrame({"Bilgi": [bt["error"]]}), go.Figure().update_layout(title="Backtest hatası", template="plotly_dark"), pd.DataFrame()
|
|
|
|
| 870 |
|
| 871 |
results_df = pd.DataFrame(bt["results"])
|
|
|
|
| 872 |
fig = go.Figure()
|
| 873 |
fig.add_trace(go.Bar(name="Model MAE", x=results_df["split"], y=results_df["mae"], marker_color="cyan"))
|
| 874 |
fig.add_trace(go.Bar(name="Baseline MAE", x=results_df["split"], y=results_df["baseline_mae"], marker_color="gray"))
|
| 875 |
+
fig.update_layout(title="Walk-Forward: MAE", xaxis_title="Split", yaxis_title="MAE (log getiri)",
|
| 876 |
+
barmode="group", template="plotly_dark")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 877 |
|
| 878 |
+
sim_df = pd.DataFrame([bt["simulation"]]) if bt.get("simulation") else pd.DataFrame()
|
| 879 |
return results_df, fig, sim_df
|
| 880 |
|
| 881 |
|
|
|
|
| 886 |
if fname.endswith(".csv"):
|
| 887 |
fpath = os.path.join(DATA_DIR, fname)
|
| 888 |
try:
|
| 889 |
+
rows.append({
|
| 890 |
+
"Dosya": fname,
|
| 891 |
+
"Boyut (KB)": round(os.path.getsize(fpath) / 1024, 2),
|
| 892 |
+
"Son Değişiklik": dt.datetime.fromtimestamp(os.path.getmtime(fpath)).strftime("%Y-%m-%d %H:%M:%S"),
|
| 893 |
+
})
|
| 894 |
except OSError:
|
| 895 |
continue
|
| 896 |
+
return pd.DataFrame(rows) if rows else pd.DataFrame(columns=["Dosya", "Boyut (KB)", "Son Değişiklik"])
|
|
|
|
|
|
|
| 897 |
|
| 898 |
|
| 899 |
def refresh_data_ui():
|
|
|
|
| 902 |
if not pipeline.featured_df.empty:
|
| 903 |
pipeline.train_model()
|
| 904 |
except Exception as e:
|
| 905 |
+
pipeline.status_message = f"Hata: {e}"
|
|
|
|
| 906 |
import traceback
|
| 907 |
traceback.print_exc()
|
| 908 |
|
| 909 |
try:
|
| 910 |
bt_table, bt_fig, sim_table = get_backtest_outputs()
|
| 911 |
+
except Exception:
|
| 912 |
+
bt_table, sim_table = pd.DataFrame(), pd.DataFrame()
|
| 913 |
bt_fig = go.Figure().update_layout(title="Backtest hatası", template="plotly_dark")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 914 |
|
| 915 |
+
def _safe(fn, default):
|
| 916 |
+
try:
|
| 917 |
+
return fn()
|
| 918 |
+
except Exception as e:
|
| 919 |
+
return pd.DataFrame({"Bilgi": [f"Hata: {e}"]}) if default is None else default
|
| 920 |
|
| 921 |
return (
|
| 922 |
pipeline.status_message,
|
| 923 |
+
_safe(get_dashboard_metrics, pd.DataFrame()),
|
| 924 |
+
_safe(get_analysis_table, pd.DataFrame()),
|
| 925 |
+
_safe(get_prediction_table, pd.DataFrame()),
|
| 926 |
+
_safe(get_arbitrage_table, pd.DataFrame()),
|
| 927 |
+
_safe(get_model_performance_plot, go.Figure().update_layout(title="—", template="plotly_dark")),
|
| 928 |
+
bt_table, bt_fig, sim_table,
|
| 929 |
+
_safe(list_csv_files, pd.DataFrame()),
|
|
|
|
|
|
|
| 930 |
)
|
| 931 |
|
| 932 |
|
| 933 |
def initial_load():
|
| 934 |
+
def _safe(fn, default):
|
| 935 |
+
try:
|
| 936 |
+
return fn()
|
| 937 |
+
except Exception:
|
| 938 |
+
return default
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
|
| 940 |
return (
|
| 941 |
+
_safe(get_dashboard_metrics, pd.DataFrame({"Metrik": ["Durum"], "Değer": ["Hazır"]})),
|
| 942 |
+
pd.DataFrame(), pd.DataFrame(), pd.DataFrame(),
|
| 943 |
+
go.Figure().update_layout(title="Yenile butonuna basın", template="plotly_dark"),
|
| 944 |
pd.DataFrame(),
|
| 945 |
+
go.Figure().update_layout(title="Veri gerekli", template="plotly_dark"),
|
| 946 |
pd.DataFrame(),
|
| 947 |
+
_safe(list_csv_files, pd.DataFrame()),
|
|
|
|
|
|
|
| 948 |
)
|
| 949 |
|
| 950 |
|
|
|
|
| 961 |
**Profesyonel Market Intelligence Dashboard** | Avrupa Sunucusu
|
| 962 |
*LightGBM tabanlı 24 saatlik fiyat tahmini ve arbitraj analizi.*
|
| 963 |
|
| 964 |
+
**Tahmin sınırı:** ±%15 · **Hedef ölçeği:** baz puan (×10000)
|
|
|
|
| 965 |
|
| 966 |
+
**⚠️ Risk Uyarısı:** Bu uygulama yatırım tavsiyesi değildir. Model tahminleri belirsizlik içerir ve kesin kazanç garantisi sunmaz.
|
| 967 |
"""
|
| 968 |
)
|
| 969 |
|
|
|
|
| 972 |
status_text = gr.Textbox(label="Durum", interactive=False, scale=2, value=pipeline.status_message)
|
| 973 |
|
| 974 |
with gr.Tabs():
|
|
|
|
| 975 |
with gr.TabItem("📊 Dashboard"):
|
| 976 |
+
dashboard_table = gr.Dataframe(label="Özet", interactive=False, wrap=True)
|
| 977 |
gr.Markdown(
|
| 978 |
+
"### Kalibrasyon\n"
|
| 979 |
+
"- Hedef baz puan ölçeğinde (×10000) → model daha stabil öğrenir\n"
|
| 980 |
+
"- Fiyat feature'ları `log1p` ile normalize edilir\n"
|
| 981 |
+
"- Tahmin ±%15 aralığına clip'lenir\n"
|
| 982 |
+
"- |tahmin| < 1% ise **belirsiz** olarak işaretlenir\n\n"
|
| 983 |
"### Riskler\n"
|
| 984 |
+
"- Model tahminleri **olasılıksaldır**\n"
|
| 985 |
+
"- Geçmiş performans gelecek garantisi değildir\n"
|
| 986 |
+
"- Spread ve likidite düşükse gerçekleşme fiyatı sapabilir"
|
| 987 |
)
|
| 988 |
|
| 989 |
with gr.TabItem("📦 100 Ürün Analizi"):
|
| 990 |
+
analysis_table = gr.Dataframe(label="Ürün Analizi", interactive=False)
|
| 991 |
|
| 992 |
with gr.TabItem("🔮 24 Saatlik Fiyat Tahmini"):
|
| 993 |
+
prediction_table = gr.Dataframe(label="Tahminler", interactive=False)
|
| 994 |
+
gr.Markdown("*⚠️ Belirsiz = model sinyali zayıf. Net getiri %2.5+%8 maliyet düşülmüş halidir.*")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 995 |
|
| 996 |
+
with gr.TabItem("🚚 Arbitraj"):
|
| 997 |
+
arbitrage_table = gr.Dataframe(label="Arbitraj Fırsatları", interactive=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 998 |
|
| 999 |
with gr.TabItem("📈 Model Performansı"):
|
| 1000 |
performance_plot = gr.Plot(label="Feature Importance")
|
| 1001 |
|
| 1002 |
+
with gr.TabItem("🧪 Backtest"):
|
| 1003 |
backtest_table = gr.Dataframe(label="Split Sonuçları", interactive=False)
|
| 1004 |
backtest_plot = gr.Plot(label="MAE Karşılaştırması")
|
| 1005 |
+
simulation_table = gr.Dataframe(label="Simülasyon", interactive=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1006 |
|
| 1007 |
with gr.TabItem("🗄️ Veri Yönetimi"):
|
| 1008 |
+
gr.Markdown("**API:** `europe.albion-online-data.com/api/v2`\n**Model:** `gos_hawk_data/models/gos_hawk_model.pkl`")
|
| 1009 |
+
csv_files = gr.Dataframe(label="CSV Dosyaları", interactive=False,
|
| 1010 |
+
headers=["Dosya", "Boyut (KB)", "Son Değişiklik"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1011 |
|
| 1012 |
refresh_btn.click(
|
| 1013 |
fn=refresh_data_ui,
|
| 1014 |
inputs=[],
|
| 1015 |
+
outputs=[status_text, dashboard_table, analysis_table, prediction_table,
|
| 1016 |
+
arbitrage_table, performance_plot, backtest_table, backtest_plot,
|
| 1017 |
+
simulation_table, csv_files],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1018 |
)
|
| 1019 |
|
| 1020 |
demo.load(
|
| 1021 |
fn=initial_load,
|
| 1022 |
inputs=[],
|
| 1023 |
+
outputs=[dashboard_table, analysis_table, prediction_table, arbitrage_table,
|
| 1024 |
+
performance_plot, backtest_table, backtest_plot, simulation_table, csv_files],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1025 |
)
|
| 1026 |
|
| 1027 |
|